clang  7.0.0
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25 
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31  for (const auto *C : S.clauses()) {
32  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
33  if (const auto *PreInit =
34  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
35  for (const auto *I : PreInit->decls()) {
36  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
37  CGF.EmitVarDecl(cast<VarDecl>(*I));
38  } else {
40  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
41  CGF.EmitAutoVarCleanups(Emission);
42  }
43  }
44  }
45  }
46  }
47  }
48  CodeGenFunction::OMPPrivateScope InlinedShareds;
49 
50  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
51  return CGF.LambdaCaptureFields.lookup(VD) ||
52  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
53  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
54  }
55 
56 public:
57  OMPLexicalScope(
59  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
60  const bool EmitPreInitStmt = true)
61  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
62  InlinedShareds(CGF) {
63  if (EmitPreInitStmt)
64  emitPreInitStmt(CGF, S);
65  if (!CapturedRegion.hasValue())
66  return;
67  assert(S.hasAssociatedStmt() &&
68  "Expected associated statement for inlined directive.");
69  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
70  for (const auto &C : CS->captures()) {
71  if (C.capturesVariable() || C.capturesVariableByCopy()) {
72  auto *VD = C.getCapturedVar();
73  assert(VD == VD->getCanonicalDecl() &&
74  "Canonical decl must be captured.");
75  DeclRefExpr DRE(
76  const_cast<VarDecl *>(VD),
77  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
78  InlinedShareds.isGlobalVarCaptured(VD)),
79  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
80  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
81  return CGF.EmitLValue(&DRE).getAddress();
82  });
83  }
84  }
85  (void)InlinedShareds.Privatize();
86  }
87 };
88 
89 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
90 /// for captured expressions.
91 class OMPParallelScope final : public OMPLexicalScope {
92  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
94  return !(isOpenMPTargetExecutionDirective(Kind) ||
97  }
98 
99 public:
100  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
101  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
102  EmitPreInitStmt(S)) {}
103 };
104 
105 /// Lexical scope for OpenMP teams construct, that handles correct codegen
106 /// for captured expressions.
107 class OMPTeamsScope final : public OMPLexicalScope {
108  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110  return !isOpenMPTargetExecutionDirective(Kind) &&
112  }
113 
114 public:
115  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
116  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
117  EmitPreInitStmt(S)) {}
118 };
119 
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124  CodeGenFunction::OMPMapVars PreCondVars;
125  for (const auto *E : S.counters()) {
126  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127  (void)PreCondVars.setVarAddr(
128  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
129  }
130  (void)PreCondVars.apply(CGF);
131  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
132  for (const auto *I : PreInits->decls())
133  CGF.EmitVarDecl(cast<VarDecl>(*I));
134  }
135  PreCondVars.restore(CGF);
136  }
137 
138 public:
139  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
140  : CodeGenFunction::RunCleanupsScope(CGF) {
141  emitPreInitStmt(CGF, S);
142  }
143 };
144 
145 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
146  CodeGenFunction::OMPPrivateScope InlinedShareds;
147 
148  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
149  return CGF.LambdaCaptureFields.lookup(VD) ||
150  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
151  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
152  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
153  }
154 
155 public:
156  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
157  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
158  InlinedShareds(CGF) {
159  for (const auto *C : S.clauses()) {
160  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
161  if (const auto *PreInit =
162  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
163  for (const auto *I : PreInit->decls()) {
164  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
165  CGF.EmitVarDecl(cast<VarDecl>(*I));
166  } else {
168  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
169  CGF.EmitAutoVarCleanups(Emission);
170  }
171  }
172  }
173  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
174  for (const Expr *E : UDP->varlists()) {
175  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
176  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
177  CGF.EmitVarDecl(*OED);
178  }
179  }
180  }
182  CGF.EmitOMPPrivateClause(S, InlinedShareds);
183  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
184  if (const Expr *E = TG->getReductionRef())
185  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
186  }
187  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
188  while (CS) {
189  for (auto &C : CS->captures()) {
190  if (C.capturesVariable() || C.capturesVariableByCopy()) {
191  auto *VD = C.getCapturedVar();
192  assert(VD == VD->getCanonicalDecl() &&
193  "Canonical decl must be captured.");
194  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
195  isCapturedVar(CGF, VD) ||
196  (CGF.CapturedStmtInfo &&
197  InlinedShareds.isGlobalVarCaptured(VD)),
199  C.getLocation());
200  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
201  return CGF.EmitLValue(&DRE).getAddress();
202  });
203  }
204  }
205  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
206  }
207  (void)InlinedShareds.Privatize();
208  }
209 };
210 
211 } // namespace
212 
214  const OMPExecutableDirective &S,
215  const RegionCodeGenTy &CodeGen);
216 
218  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
219  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
220  OrigVD = OrigVD->getCanonicalDecl();
221  bool IsCaptured =
222  LambdaCaptureFields.lookup(OrigVD) ||
223  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
224  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
225  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
226  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
227  return EmitLValue(&DRE);
228  }
229  }
230  return EmitLValue(E);
231 }
232 
234  ASTContext &C = getContext();
235  llvm::Value *Size = nullptr;
236  auto SizeInChars = C.getTypeSizeInChars(Ty);
237  if (SizeInChars.isZero()) {
238  // getTypeSizeInChars() returns 0 for a VLA.
239  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
240  VlaSizePair VlaSize = getVLASize(VAT);
241  Ty = VlaSize.Type;
242  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
243  : VlaSize.NumElts;
244  }
245  SizeInChars = C.getTypeSizeInChars(Ty);
246  if (SizeInChars.isZero())
247  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
249  }
250  return CGM.getSize(SizeInChars);
251 }
252 
254  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
255  const RecordDecl *RD = S.getCapturedRecordDecl();
256  auto CurField = RD->field_begin();
257  auto CurCap = S.captures().begin();
259  E = S.capture_init_end();
260  I != E; ++I, ++CurField, ++CurCap) {
261  if (CurField->hasCapturedVLAType()) {
262  const VariableArrayType *VAT = CurField->getCapturedVLAType();
263  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
264  CapturedVars.push_back(Val);
265  } else if (CurCap->capturesThis()) {
266  CapturedVars.push_back(CXXThisValue);
267  } else if (CurCap->capturesVariableByCopy()) {
268  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
269 
270  // If the field is not a pointer, we need to save the actual value
271  // and load it as a void pointer.
272  if (!CurField->getType()->isAnyPointerType()) {
273  ASTContext &Ctx = getContext();
274  Address DstAddr = CreateMemTemp(
275  Ctx.getUIntPtrType(),
276  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
277  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
278 
279  llvm::Value *SrcAddrVal = EmitScalarConversion(
280  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
281  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
282  LValue SrcLV =
283  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
284 
285  // Store the value using the source type pointer.
286  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
287 
288  // Load the value using the destination type pointer.
289  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
290  }
291  CapturedVars.push_back(CV);
292  } else {
293  assert(CurCap->capturesVariable() && "Expected capture by reference.");
294  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
295  }
296  }
297 }
298 
300  QualType DstType, StringRef Name,
301  LValue AddrLV,
302  bool isReferenceType = false) {
303  ASTContext &Ctx = CGF.getContext();
304 
305  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
306  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
307  Ctx.getPointerType(DstType), Loc);
308  Address TmpAddr =
309  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
310  .getAddress();
311 
312  // If we are dealing with references we need to return the address of the
313  // reference instead of the reference of the value.
314  if (isReferenceType) {
315  QualType RefType = Ctx.getLValueReferenceType(DstType);
316  llvm::Value *RefVal = TmpAddr.getPointer();
317  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
318  LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
319  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
320  }
321 
322  return TmpAddr;
323 }
324 
326  if (T->isLValueReferenceType())
327  return C.getLValueReferenceType(
329  /*SpelledAsLValue=*/false);
330  if (T->isPointerType())
332  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
333  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
334  return getCanonicalParamType(C, VLA->getElementType());
335  if (!A->isVariablyModifiedType())
336  return C.getCanonicalType(T);
337  }
338  return C.getCanonicalParamType(T);
339 }
340 
341 namespace {
342  /// Contains required data for proper outlined function codegen.
343  struct FunctionOptions {
344  /// Captured statement for which the function is generated.
345  const CapturedStmt *S = nullptr;
346  /// true if cast to/from UIntPtr is required for variables captured by
347  /// value.
348  const bool UIntPtrCastRequired = true;
349  /// true if only casted arguments must be registered as local args or VLA
350  /// sizes.
351  const bool RegisterCastedArgsOnly = false;
352  /// Name of the generated function.
353  const StringRef FunctionName;
354  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
355  bool RegisterCastedArgsOnly,
356  StringRef FunctionName)
357  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
358  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
359  FunctionName(FunctionName) {}
360  };
361 }
362 
363 static llvm::Function *emitOutlinedFunctionPrologue(
364  CodeGenFunction &CGF, FunctionArgList &Args,
365  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
366  &LocalAddrs,
367  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
368  &VLASizes,
369  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
370  const CapturedDecl *CD = FO.S->getCapturedDecl();
371  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
372  assert(CD->hasBody() && "missing CapturedDecl body");
373 
374  CXXThisValue = nullptr;
375  // Build the argument list.
376  CodeGenModule &CGM = CGF.CGM;
377  ASTContext &Ctx = CGM.getContext();
378  FunctionArgList TargetArgs;
379  Args.append(CD->param_begin(),
380  std::next(CD->param_begin(), CD->getContextParamPosition()));
381  TargetArgs.append(
382  CD->param_begin(),
383  std::next(CD->param_begin(), CD->getContextParamPosition()));
384  auto I = FO.S->captures().begin();
385  FunctionDecl *DebugFunctionDecl = nullptr;
386  if (!FO.UIntPtrCastRequired) {
388  DebugFunctionDecl = FunctionDecl::Create(
389  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(),
392  Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
393  SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
394  }
395  for (const FieldDecl *FD : RD->fields()) {
396  QualType ArgType = FD->getType();
397  IdentifierInfo *II = nullptr;
398  VarDecl *CapVar = nullptr;
399 
400  // If this is a capture by copy and the type is not a pointer, the outlined
401  // function argument type should be uintptr and the value properly casted to
402  // uintptr. This is necessary given that the runtime library is only able to
403  // deal with pointers. We can pass in the same way the VLA type sizes to the
404  // outlined function.
405  if (FO.UIntPtrCastRequired &&
406  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
407  I->capturesVariableArrayType()))
408  ArgType = Ctx.getUIntPtrType();
409 
410  if (I->capturesVariable() || I->capturesVariableByCopy()) {
411  CapVar = I->getCapturedVar();
412  II = CapVar->getIdentifier();
413  } else if (I->capturesThis()) {
414  II = &Ctx.Idents.get("this");
415  } else {
416  assert(I->capturesVariableArrayType());
417  II = &Ctx.Idents.get("vla");
418  }
419  if (ArgType->isVariablyModifiedType())
420  ArgType = getCanonicalParamType(Ctx, ArgType);
421  VarDecl *Arg;
422  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
423  Arg = ParmVarDecl::Create(
424  Ctx, DebugFunctionDecl,
425  CapVar ? CapVar->getLocStart() : FD->getLocStart(),
426  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
427  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
428  } else {
429  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
430  II, ArgType, ImplicitParamDecl::Other);
431  }
432  Args.emplace_back(Arg);
433  // Do not cast arguments if we emit function with non-original types.
434  TargetArgs.emplace_back(
435  FO.UIntPtrCastRequired
436  ? Arg
437  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
438  ++I;
439  }
440  Args.append(
441  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
442  CD->param_end());
443  TargetArgs.append(
444  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
445  CD->param_end());
446 
447  // Create the function declaration.
448  const CGFunctionInfo &FuncInfo =
449  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
450  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
451 
452  auto *F =
454  FO.FunctionName, &CGM.getModule());
455  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
456  if (CD->isNothrow())
457  F->setDoesNotThrow();
458  F->setDoesNotRecurse();
459 
460  // Generate the function.
461  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
462  FO.S->getLocStart(), CD->getBody()->getLocStart());
463  unsigned Cnt = CD->getContextParamPosition();
464  I = FO.S->captures().begin();
465  for (const FieldDecl *FD : RD->fields()) {
466  // Do not map arguments if we emit function with non-original types.
467  Address LocalAddr(Address::invalid());
468  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
469  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
470  TargetArgs[Cnt]);
471  } else {
472  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
473  }
474  // If we are capturing a pointer by copy we don't need to do anything, just
475  // use the value that we get from the arguments.
476  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
477  const VarDecl *CurVD = I->getCapturedVar();
478  // If the variable is a reference we need to materialize it here.
479  if (CurVD->getType()->isReferenceType()) {
480  Address RefAddr = CGF.CreateMemTemp(
481  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
482  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
483  /*Volatile=*/false, CurVD->getType());
484  LocalAddr = RefAddr;
485  }
486  if (!FO.RegisterCastedArgsOnly)
487  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
488  ++Cnt;
489  ++I;
490  continue;
491  }
492 
493  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
495  if (FD->hasCapturedVLAType()) {
496  if (FO.UIntPtrCastRequired) {
497  ArgLVal = CGF.MakeAddrLValue(
498  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
499  Args[Cnt]->getName(), ArgLVal),
501  }
502  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
503  const VariableArrayType *VAT = FD->getCapturedVLAType();
504  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
505  } else if (I->capturesVariable()) {
506  const VarDecl *Var = I->getCapturedVar();
507  QualType VarTy = Var->getType();
508  Address ArgAddr = ArgLVal.getAddress();
509  if (!VarTy->isReferenceType()) {
510  if (ArgLVal.getType()->isLValueReferenceType()) {
511  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
512  } else if (!VarTy->isVariablyModifiedType() ||
513  !VarTy->isPointerType()) {
514  assert(ArgLVal.getType()->isPointerType());
515  ArgAddr = CGF.EmitLoadOfPointer(
516  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
517  }
518  }
519  if (!FO.RegisterCastedArgsOnly) {
520  LocalAddrs.insert(
521  {Args[Cnt],
522  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
523  }
524  } else if (I->capturesVariableByCopy()) {
525  assert(!FD->getType()->isAnyPointerType() &&
526  "Not expecting a captured pointer.");
527  const VarDecl *Var = I->getCapturedVar();
528  QualType VarTy = Var->getType();
529  LocalAddrs.insert(
530  {Args[Cnt],
531  {Var, FO.UIntPtrCastRequired
532  ? castValueFromUintptr(CGF, I->getLocation(),
533  FD->getType(), Args[Cnt]->getName(),
534  ArgLVal, VarTy->isReferenceType())
535  : ArgLVal.getAddress()}});
536  } else {
537  // If 'this' is captured, load it into CXXThisValue.
538  assert(I->capturesThis());
539  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
540  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
541  }
542  ++Cnt;
543  ++I;
544  }
545 
546  return F;
547 }
548 
549 llvm::Function *
551  assert(
552  CapturedStmtInfo &&
553  "CapturedStmtInfo should be set when generating the captured function");
554  const CapturedDecl *CD = S.getCapturedDecl();
555  // Build the argument list.
556  bool NeedWrapperFunction =
557  getDebugInfo() &&
558  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
559  FunctionArgList Args;
560  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
561  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
562  SmallString<256> Buffer;
563  llvm::raw_svector_ostream Out(Buffer);
564  Out << CapturedStmtInfo->getHelperName();
565  if (NeedWrapperFunction)
566  Out << "_debug__";
567  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
568  Out.str());
569  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
570  VLASizes, CXXThisValue, FO);
571  for (const auto &LocalAddrPair : LocalAddrs) {
572  if (LocalAddrPair.second.first) {
573  setAddrOfLocalVar(LocalAddrPair.second.first,
574  LocalAddrPair.second.second);
575  }
576  }
577  for (const auto &VLASizePair : VLASizes)
578  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
579  PGO.assignRegionCounters(GlobalDecl(CD), F);
580  CapturedStmtInfo->EmitBody(*this, CD->getBody());
581  FinishFunction(CD->getBodyRBrace());
582  if (!NeedWrapperFunction)
583  return F;
584 
585  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
586  /*RegisterCastedArgsOnly=*/true,
587  CapturedStmtInfo->getHelperName());
588  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
589  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
590  Args.clear();
591  LocalAddrs.clear();
592  VLASizes.clear();
593  llvm::Function *WrapperF =
594  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
595  WrapperCGF.CXXThisValue, WrapperFO);
597  for (const auto *Arg : Args) {
599  auto I = LocalAddrs.find(Arg);
600  if (I != LocalAddrs.end()) {
601  LValue LV = WrapperCGF.MakeAddrLValue(
602  I->second.second,
603  I->second.first ? I->second.first->getType() : Arg->getType(),
605  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart());
606  } else {
607  auto EI = VLASizes.find(Arg);
608  if (EI != VLASizes.end()) {
609  CallArg = EI->second.second;
610  } else {
611  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
612  Arg->getType(),
614  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart());
615  }
616  }
617  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
618  }
619  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(),
620  F, CallArgs);
621  WrapperCGF.FinishFunction();
622  return WrapperF;
623 }
624 
625 //===----------------------------------------------------------------------===//
626 // OpenMP Directive Emission
627 //===----------------------------------------------------------------------===//
629  Address DestAddr, Address SrcAddr, QualType OriginalType,
630  const llvm::function_ref<void(Address, Address)> CopyGen) {
631  // Perform element-by-element initialization.
632  QualType ElementTy;
633 
634  // Drill down to the base element type on both arrays.
635  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
636  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
637  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
638 
639  llvm::Value *SrcBegin = SrcAddr.getPointer();
640  llvm::Value *DestBegin = DestAddr.getPointer();
641  // Cast from pointer to array type to pointer to single element.
642  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
643  // The basic structure here is a while-do loop.
644  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
645  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
646  llvm::Value *IsEmpty =
647  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
648  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
649 
650  // Enter the loop body, making that address the current address.
651  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
652  EmitBlock(BodyBB);
653 
654  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
655 
656  llvm::PHINode *SrcElementPHI =
657  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
658  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
659  Address SrcElementCurrent =
660  Address(SrcElementPHI,
661  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
662 
663  llvm::PHINode *DestElementPHI =
664  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
665  DestElementPHI->addIncoming(DestBegin, EntryBB);
666  Address DestElementCurrent =
667  Address(DestElementPHI,
668  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
669 
670  // Emit copy.
671  CopyGen(DestElementCurrent, SrcElementCurrent);
672 
673  // Shift the address forward by one element.
674  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
675  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
676  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
677  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
678  // Check whether we've reached the end.
679  llvm::Value *Done =
680  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
681  Builder.CreateCondBr(Done, DoneBB, BodyBB);
682  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
683  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
684 
685  // Done.
686  EmitBlock(DoneBB, /*IsFinished=*/true);
687 }
688 
689 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
690  Address SrcAddr, const VarDecl *DestVD,
691  const VarDecl *SrcVD, const Expr *Copy) {
692  if (OriginalType->isArrayType()) {
693  const auto *BO = dyn_cast<BinaryOperator>(Copy);
694  if (BO && BO->getOpcode() == BO_Assign) {
695  // Perform simple memcpy for simple copying.
696  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
697  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
698  EmitAggregateAssign(Dest, Src, OriginalType);
699  } else {
700  // For arrays with complex element types perform element by element
701  // copying.
702  EmitOMPAggregateAssign(
703  DestAddr, SrcAddr, OriginalType,
704  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705  // Working with the single array element, so have to remap
706  // destination and source variables to corresponding array
707  // elements.
709  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
710  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
711  (void)Remap.Privatize();
712  EmitIgnoredExpr(Copy);
713  });
714  }
715  } else {
716  // Remap pseudo source variable to private copy.
718  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
719  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
720  (void)Remap.Privatize();
721  // Emit copying of the whole variable.
722  EmitIgnoredExpr(Copy);
723  }
724 }
725 
727  OMPPrivateScope &PrivateScope) {
728  if (!HaveInsertPoint())
729  return false;
730  bool FirstprivateIsLastprivate = false;
731  llvm::DenseSet<const VarDecl *> Lastprivates;
732  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
733  for (const auto *D : C->varlists())
734  Lastprivates.insert(
735  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
736  }
737  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
739  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
740  // Force emission of the firstprivate copy if the directive does not emit
741  // outlined function, like omp for, omp simd, omp distribute etc.
742  bool MustEmitFirstprivateCopy =
743  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
744  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
745  auto IRef = C->varlist_begin();
746  auto InitsRef = C->inits().begin();
747  for (const Expr *IInit : C->private_copies()) {
748  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
749  bool ThisFirstprivateIsLastprivate =
750  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
751  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
752  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
753  !FD->getType()->isReferenceType()) {
754  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
755  ++IRef;
756  ++InitsRef;
757  continue;
758  }
759  FirstprivateIsLastprivate =
760  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
761  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
762  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
763  const auto *VDInit =
764  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
765  bool IsRegistered;
766  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
767  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
768  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
769  LValue OriginalLVal = EmitLValue(&DRE);
770  QualType Type = VD->getType();
771  if (Type->isArrayType()) {
772  // Emit VarDecl with copy init for arrays.
773  // Get the address of the original variable captured in current
774  // captured region.
775  IsRegistered = PrivateScope.addPrivate(
776  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
777  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
778  const Expr *Init = VD->getInit();
779  if (!isa<CXXConstructExpr>(Init) ||
780  isTrivialInitializer(Init)) {
781  // Perform simple memcpy.
782  LValue Dest =
783  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
784  EmitAggregateAssign(Dest, OriginalLVal, Type);
785  } else {
786  EmitOMPAggregateAssign(
787  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
788  Type,
789  [this, VDInit, Init](Address DestElement,
790  Address SrcElement) {
791  // Clean up any temporaries needed by the
792  // initialization.
793  RunCleanupsScope InitScope(*this);
794  // Emit initialization for single element.
795  setAddrOfLocalVar(VDInit, SrcElement);
796  EmitAnyExprToMem(Init, DestElement,
797  Init->getType().getQualifiers(),
798  /*IsInitializer*/ false);
799  LocalDeclMap.erase(VDInit);
800  });
801  }
802  EmitAutoVarCleanups(Emission);
803  return Emission.getAllocatedAddress();
804  });
805  } else {
806  Address OriginalAddr = OriginalLVal.getAddress();
807  IsRegistered = PrivateScope.addPrivate(
808  OrigVD, [this, VDInit, OriginalAddr, VD]() {
809  // Emit private VarDecl with copy init.
810  // Remap temp VDInit variable to the address of the original
811  // variable (for proper handling of captured global variables).
812  setAddrOfLocalVar(VDInit, OriginalAddr);
813  EmitDecl(*VD);
814  LocalDeclMap.erase(VDInit);
815  return GetAddrOfLocalVar(VD);
816  });
817  }
818  assert(IsRegistered &&
819  "firstprivate var already registered as private");
820  // Silence the warning about unused variable.
821  (void)IsRegistered;
822  }
823  ++IRef;
824  ++InitsRef;
825  }
826  }
827  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
828 }
829 
831  const OMPExecutableDirective &D,
832  CodeGenFunction::OMPPrivateScope &PrivateScope) {
833  if (!HaveInsertPoint())
834  return;
835  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
836  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
837  auto IRef = C->varlist_begin();
838  for (const Expr *IInit : C->private_copies()) {
839  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
840  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
841  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
842  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
843  // Emit private VarDecl with copy init.
844  EmitDecl(*VD);
845  return GetAddrOfLocalVar(VD);
846  });
847  assert(IsRegistered && "private var already registered as private");
848  // Silence the warning about unused variable.
849  (void)IsRegistered;
850  }
851  ++IRef;
852  }
853  }
854 }
855 
857  if (!HaveInsertPoint())
858  return false;
859  // threadprivate_var1 = master_threadprivate_var1;
860  // operator=(threadprivate_var2, master_threadprivate_var2);
861  // ...
862  // __kmpc_barrier(&loc, global_tid);
864  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
865  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
866  auto IRef = C->varlist_begin();
867  auto ISrcRef = C->source_exprs().begin();
868  auto IDestRef = C->destination_exprs().begin();
869  for (const Expr *AssignOp : C->assignment_ops()) {
870  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
871  QualType Type = VD->getType();
872  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
873  // Get the address of the master variable. If we are emitting code with
874  // TLS support, the address is passed from the master as field in the
875  // captured declaration.
876  Address MasterAddr = Address::invalid();
877  if (getLangOpts().OpenMPUseTLS &&
878  getContext().getTargetInfo().isTLSSupported()) {
879  assert(CapturedStmtInfo->lookup(VD) &&
880  "Copyin threadprivates should have been captured!");
881  DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
882  VK_LValue, (*IRef)->getExprLoc());
883  MasterAddr = EmitLValue(&DRE).getAddress();
884  LocalDeclMap.erase(VD);
885  } else {
886  MasterAddr =
887  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
888  : CGM.GetAddrOfGlobal(VD),
889  getContext().getDeclAlign(VD));
890  }
891  // Get the address of the threadprivate variable.
892  Address PrivateAddr = EmitLValue(*IRef).getAddress();
893  if (CopiedVars.size() == 1) {
894  // At first check if current thread is a master thread. If it is, no
895  // need to copy data.
896  CopyBegin = createBasicBlock("copyin.not.master");
897  CopyEnd = createBasicBlock("copyin.not.master.end");
898  Builder.CreateCondBr(
899  Builder.CreateICmpNE(
900  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
901  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
902  CGM.IntPtrTy)),
903  CopyBegin, CopyEnd);
904  EmitBlock(CopyBegin);
905  }
906  const auto *SrcVD =
907  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
908  const auto *DestVD =
909  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
910  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
911  }
912  ++IRef;
913  ++ISrcRef;
914  ++IDestRef;
915  }
916  }
917  if (CopyEnd) {
918  // Exit out of copying procedure for non-master thread.
919  EmitBlock(CopyEnd, /*IsFinished=*/true);
920  return true;
921  }
922  return false;
923 }
924 
926  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
927  if (!HaveInsertPoint())
928  return false;
929  bool HasAtLeastOneLastprivate = false;
932  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
933  for (const Expr *C : LoopDirective->counters()) {
934  SIMDLCVs.insert(
935  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
936  }
937  }
938  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
939  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
940  HasAtLeastOneLastprivate = true;
942  !getLangOpts().OpenMPSimd)
943  break;
944  auto IRef = C->varlist_begin();
945  auto IDestRef = C->destination_exprs().begin();
946  for (const Expr *IInit : C->private_copies()) {
947  // Keep the address of the original variable for future update at the end
948  // of the loop.
949  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
950  // Taskloops do not require additional initialization, it is done in
951  // runtime support library.
952  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
953  const auto *DestVD =
954  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
955  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
956  DeclRefExpr DRE(
957  const_cast<VarDecl *>(OrigVD),
958  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
959  OrigVD) != nullptr,
960  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
961  return EmitLValue(&DRE).getAddress();
962  });
963  // Check if the variable is also a firstprivate: in this case IInit is
964  // not generated. Initialization of this variable will happen in codegen
965  // for 'firstprivate' clause.
966  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
967  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
968  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
969  // Emit private VarDecl with copy init.
970  EmitDecl(*VD);
971  return GetAddrOfLocalVar(VD);
972  });
973  assert(IsRegistered &&
974  "lastprivate var already registered as private");
975  (void)IsRegistered;
976  }
977  }
978  ++IRef;
979  ++IDestRef;
980  }
981  }
982  return HasAtLeastOneLastprivate;
983 }
984 
986  const OMPExecutableDirective &D, bool NoFinals,
987  llvm::Value *IsLastIterCond) {
988  if (!HaveInsertPoint())
989  return;
990  // Emit following code:
991  // if (<IsLastIterCond>) {
992  // orig_var1 = private_orig_var1;
993  // ...
994  // orig_varn = private_orig_varn;
995  // }
996  llvm::BasicBlock *ThenBB = nullptr;
997  llvm::BasicBlock *DoneBB = nullptr;
998  if (IsLastIterCond) {
999  ThenBB = createBasicBlock(".omp.lastprivate.then");
1000  DoneBB = createBasicBlock(".omp.lastprivate.done");
1001  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1002  EmitBlock(ThenBB);
1003  }
1004  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1005  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1006  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1007  auto IC = LoopDirective->counters().begin();
1008  for (const Expr *F : LoopDirective->finals()) {
1009  const auto *D =
1010  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1011  if (NoFinals)
1012  AlreadyEmittedVars.insert(D);
1013  else
1014  LoopCountersAndUpdates[D] = F;
1015  ++IC;
1016  }
1017  }
1018  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1019  auto IRef = C->varlist_begin();
1020  auto ISrcRef = C->source_exprs().begin();
1021  auto IDestRef = C->destination_exprs().begin();
1022  for (const Expr *AssignOp : C->assignment_ops()) {
1023  const auto *PrivateVD =
1024  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1025  QualType Type = PrivateVD->getType();
1026  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1027  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1028  // If lastprivate variable is a loop control variable for loop-based
1029  // directive, update its value before copyin back to original
1030  // variable.
1031  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1032  EmitIgnoredExpr(FinalExpr);
1033  const auto *SrcVD =
1034  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1035  const auto *DestVD =
1036  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1037  // Get the address of the original variable.
1038  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1039  // Get the address of the private variable.
1040  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1041  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1042  PrivateAddr =
1043  Address(Builder.CreateLoad(PrivateAddr),
1044  getNaturalTypeAlignment(RefTy->getPointeeType()));
1045  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1046  }
1047  ++IRef;
1048  ++ISrcRef;
1049  ++IDestRef;
1050  }
1051  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1052  EmitIgnoredExpr(PostUpdate);
1053  }
1054  if (IsLastIterCond)
1055  EmitBlock(DoneBB, /*IsFinished=*/true);
1056 }
1057 
1059  const OMPExecutableDirective &D,
1060  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1061  if (!HaveInsertPoint())
1062  return;
1065  SmallVector<const Expr *, 4> ReductionOps;
1068  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1069  auto IPriv = C->privates().begin();
1070  auto IRed = C->reduction_ops().begin();
1071  auto ILHS = C->lhs_exprs().begin();
1072  auto IRHS = C->rhs_exprs().begin();
1073  for (const Expr *Ref : C->varlists()) {
1074  Shareds.emplace_back(Ref);
1075  Privates.emplace_back(*IPriv);
1076  ReductionOps.emplace_back(*IRed);
1077  LHSs.emplace_back(*ILHS);
1078  RHSs.emplace_back(*IRHS);
1079  std::advance(IPriv, 1);
1080  std::advance(IRed, 1);
1081  std::advance(ILHS, 1);
1082  std::advance(IRHS, 1);
1083  }
1084  }
1085  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1086  unsigned Count = 0;
1087  auto ILHS = LHSs.begin();
1088  auto IRHS = RHSs.begin();
1089  auto IPriv = Privates.begin();
1090  for (const Expr *IRef : Shareds) {
1091  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1092  // Emit private VarDecl with reduction init.
1093  RedCG.emitSharedLValue(*this, Count);
1094  RedCG.emitAggregateType(*this, Count);
1095  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1096  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1097  RedCG.getSharedLValue(Count),
1098  [&Emission](CodeGenFunction &CGF) {
1099  CGF.EmitAutoVarInit(Emission);
1100  return true;
1101  });
1102  EmitAutoVarCleanups(Emission);
1103  Address BaseAddr = RedCG.adjustPrivateAddress(
1104  *this, Count, Emission.getAllocatedAddress());
1105  bool IsRegistered = PrivateScope.addPrivate(
1106  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1107  assert(IsRegistered && "private var already registered as private");
1108  // Silence the warning about unused variable.
1109  (void)IsRegistered;
1110 
1111  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1112  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1113  QualType Type = PrivateVD->getType();
1114  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1115  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1116  // Store the address of the original variable associated with the LHS
1117  // implicit variable.
1118  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1119  return RedCG.getSharedLValue(Count).getAddress();
1120  });
1121  PrivateScope.addPrivate(
1122  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1123  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1124  isa<ArraySubscriptExpr>(IRef)) {
1125  // Store the address of the original variable associated with the LHS
1126  // implicit variable.
1127  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1128  return RedCG.getSharedLValue(Count).getAddress();
1129  });
1130  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1131  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1132  ConvertTypeForMem(RHSVD->getType()),
1133  "rhs.begin");
1134  });
1135  } else {
1136  QualType Type = PrivateVD->getType();
1137  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1138  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1139  // Store the address of the original variable associated with the LHS
1140  // implicit variable.
1141  if (IsArray) {
1142  OriginalAddr = Builder.CreateElementBitCast(
1143  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1144  }
1145  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1146  PrivateScope.addPrivate(
1147  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1148  return IsArray
1149  ? Builder.CreateElementBitCast(
1150  GetAddrOfLocalVar(PrivateVD),
1151  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1152  : GetAddrOfLocalVar(PrivateVD);
1153  });
1154  }
1155  ++ILHS;
1156  ++IRHS;
1157  ++IPriv;
1158  ++Count;
1159  }
1160 }
1161 
1163  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1164  if (!HaveInsertPoint())
1165  return;
1170  bool HasAtLeastOneReduction = false;
1171  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1172  HasAtLeastOneReduction = true;
1173  Privates.append(C->privates().begin(), C->privates().end());
1174  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1175  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1176  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1177  }
1178  if (HasAtLeastOneReduction) {
1179  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1181  ReductionKind == OMPD_simd;
1182  bool SimpleReduction = ReductionKind == OMPD_simd;
1183  // Emit nowait reduction if nowait clause is present or directive is a
1184  // parallel directive (it always has implicit barrier).
1185  CGM.getOpenMPRuntime().emitReduction(
1186  *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
1187  {WithNowait, SimpleReduction, ReductionKind});
1188  }
1189 }
1190 
1192  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1193  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1194  if (!CGF.HaveInsertPoint())
1195  return;
1196  llvm::BasicBlock *DoneBB = nullptr;
1197  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1198  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1199  if (!DoneBB) {
1200  if (llvm::Value *Cond = CondGen(CGF)) {
1201  // If the first post-update expression is found, emit conditional
1202  // block if it was requested.
1203  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1204  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1205  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1206  CGF.EmitBlock(ThenBB);
1207  }
1208  }
1209  CGF.EmitIgnoredExpr(PostUpdate);
1210  }
1211  }
1212  if (DoneBB)
1213  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1214 }
1215 
1216 namespace {
1217 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1218 /// parallel function. This is necessary for combined constructs such as
1219 /// 'distribute parallel for'
1220 typedef llvm::function_ref<void(CodeGenFunction &,
1221  const OMPExecutableDirective &,
1223  CodeGenBoundParametersTy;
1224 } // anonymous namespace
1225 
1227  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1228  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1229  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1230  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1231  llvm::Value *OutlinedFn =
1232  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1233  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1234  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1235  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1236  llvm::Value *NumThreads =
1237  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1238  /*IgnoreResultAssign=*/true);
1239  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1240  CGF, NumThreads, NumThreadsClause->getLocStart());
1241  }
1242  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1243  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1244  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1245  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1246  }
1247  const Expr *IfCond = nullptr;
1248  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1249  if (C->getNameModifier() == OMPD_unknown ||
1250  C->getNameModifier() == OMPD_parallel) {
1251  IfCond = C->getCondition();
1252  break;
1253  }
1254  }
1255 
1256  OMPParallelScope Scope(CGF, S);
1258  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1259  // lower and upper bounds with the pragma 'for' chunking mechanism.
1260  // The following lambda takes care of appending the lower and upper bound
1261  // parameters when necessary
1262  CodeGenBoundParameters(CGF, S, CapturedVars);
1263  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1264  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
1265  CapturedVars, IfCond);
1266 }
1267 
1269  const OMPExecutableDirective &,
1271 
1273  // Emit parallel region as a standalone region.
1274  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1275  Action.Enter(CGF);
1276  OMPPrivateScope PrivateScope(CGF);
1277  bool Copyins = CGF.EmitOMPCopyinClause(S);
1278  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1279  if (Copyins) {
1280  // Emit implicit barrier to synchronize threads and avoid data races on
1281  // propagation master's thread values of threadprivate variables to local
1282  // instances of that variables of all other implicit threads.
1283  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1284  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1285  /*ForceSimpleCall=*/true);
1286  }
1287  CGF.EmitOMPPrivateClause(S, PrivateScope);
1288  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1289  (void)PrivateScope.Privatize();
1290  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1291  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1292  };
1293  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1296  [](CodeGenFunction &) { return nullptr; });
1297 }
1298 
1300  JumpDest LoopExit) {
1301  RunCleanupsScope BodyScope(*this);
1302  // Update counters values on current iteration.
1303  for (const Expr *UE : D.updates())
1304  EmitIgnoredExpr(UE);
1305  // Update the linear variables.
1306  // In distribute directives only loop counters may be marked as linear, no
1307  // need to generate the code for them.
1309  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1310  for (const Expr *UE : C->updates())
1311  EmitIgnoredExpr(UE);
1312  }
1313  }
1314 
1315  // On a continue in the body, jump to the end.
1316  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1317  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1318  // Emit loop body.
1319  EmitStmt(D.getBody());
1320  // The end (updates/cleanups).
1321  EmitBlock(Continue.getBlock());
1322  BreakContinueStack.pop_back();
1323 }
1324 
1326  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1327  const Expr *IncExpr,
1328  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1329  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1330  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1331 
1332  // Start the loop with a block that tests the condition.
1333  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1334  EmitBlock(CondBlock);
1335  const SourceRange R = S.getSourceRange();
1336  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1337  SourceLocToDebugLoc(R.getEnd()));
1338 
1339  // If there are any cleanups between here and the loop-exit scope,
1340  // create a block to stage a loop exit along.
1341  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1342  if (RequiresCleanup)
1343  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1344 
1345  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1346 
1347  // Emit condition.
1348  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1349  if (ExitBlock != LoopExit.getBlock()) {
1350  EmitBlock(ExitBlock);
1351  EmitBranchThroughCleanup(LoopExit);
1352  }
1353 
1354  EmitBlock(LoopBody);
1355  incrementProfileCounter(&S);
1356 
1357  // Create a block for the increment.
1358  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1359  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1360 
1361  BodyGen(*this);
1362 
1363  // Emit "IV = IV + 1" and a back-edge to the condition block.
1364  EmitBlock(Continue.getBlock());
1365  EmitIgnoredExpr(IncExpr);
1366  PostIncGen(*this);
1367  BreakContinueStack.pop_back();
1368  EmitBranch(CondBlock);
1369  LoopStack.pop();
1370  // Emit the fall-through block.
1371  EmitBlock(LoopExit.getBlock());
1372 }
1373 
1375  if (!HaveInsertPoint())
1376  return false;
1377  // Emit inits for the linear variables.
1378  bool HasLinears = false;
1379  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1380  for (const Expr *Init : C->inits()) {
1381  HasLinears = true;
1382  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1383  if (const auto *Ref =
1384  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1385  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1386  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1387  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1388  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1389  VD->getInit()->getType(), VK_LValue,
1390  VD->getInit()->getExprLoc());
1391  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1392  VD->getType()),
1393  /*capturedByInit=*/false);
1394  EmitAutoVarCleanups(Emission);
1395  } else {
1396  EmitVarDecl(*VD);
1397  }
1398  }
1399  // Emit the linear steps for the linear clauses.
1400  // If a step is not constant, it is pre-calculated before the loop.
1401  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1402  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1403  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1404  // Emit calculation of the linear step.
1405  EmitIgnoredExpr(CS);
1406  }
1407  }
1408  return HasLinears;
1409 }
1410 
1412  const OMPLoopDirective &D,
1413  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1414  if (!HaveInsertPoint())
1415  return;
1416  llvm::BasicBlock *DoneBB = nullptr;
1417  // Emit the final values of the linear variables.
1418  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1419  auto IC = C->varlist_begin();
1420  for (const Expr *F : C->finals()) {
1421  if (!DoneBB) {
1422  if (llvm::Value *Cond = CondGen(*this)) {
1423  // If the first post-update expression is found, emit conditional
1424  // block if it was requested.
1425  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1426  DoneBB = createBasicBlock(".omp.linear.pu.done");
1427  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1428  EmitBlock(ThenBB);
1429  }
1430  }
1431  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1432  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1433  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1434  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1435  Address OrigAddr = EmitLValue(&DRE).getAddress();
1436  CodeGenFunction::OMPPrivateScope VarScope(*this);
1437  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1438  (void)VarScope.Privatize();
1439  EmitIgnoredExpr(F);
1440  ++IC;
1441  }
1442  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1443  EmitIgnoredExpr(PostUpdate);
1444  }
1445  if (DoneBB)
1446  EmitBlock(DoneBB, /*IsFinished=*/true);
1447 }
1448 
1450  const OMPExecutableDirective &D) {
1451  if (!CGF.HaveInsertPoint())
1452  return;
1453  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1454  unsigned ClauseAlignment = 0;
1455  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1456  auto *AlignmentCI =
1457  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1458  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1459  }
1460  for (const Expr *E : Clause->varlists()) {
1461  unsigned Alignment = ClauseAlignment;
1462  if (Alignment == 0) {
1463  // OpenMP [2.8.1, Description]
1464  // If no optional parameter is specified, implementation-defined default
1465  // alignments for SIMD instructions on the target platforms are assumed.
1466  Alignment =
1467  CGF.getContext()
1469  E->getType()->getPointeeType()))
1470  .getQuantity();
1471  }
1472  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1473  "alignment is not power of 2");
1474  if (Alignment != 0) {
1475  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1476  CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1477  }
1478  }
1479  }
1480 }
1481 
1483  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1484  if (!HaveInsertPoint())
1485  return;
1486  auto I = S.private_counters().begin();
1487  for (const Expr *E : S.counters()) {
1488  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1489  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1490  // Emit var without initialization.
1491  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1492  EmitAutoVarCleanups(VarEmission);
1493  LocalDeclMap.erase(PrivateVD);
1494  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1495  return VarEmission.getAllocatedAddress();
1496  });
1497  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1498  VD->hasGlobalStorage()) {
1499  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1500  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1501  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1502  E->getType(), VK_LValue, E->getExprLoc());
1503  return EmitLValue(&DRE).getAddress();
1504  });
1505  } else {
1506  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1507  return VarEmission.getAllocatedAddress();
1508  });
1509  }
1510  ++I;
1511  }
1512  // Privatize extra loop counters used in loops for ordered(n) clauses.
1513  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1514  if (!C->getNumForLoops())
1515  continue;
1516  for (unsigned I = S.getCollapsedNumber(),
1517  E = C->getLoopNumIterations().size();
1518  I < E; ++I) {
1519  const auto *DRE = cast<DeclRefExpr>(C->getLoopCunter(I));
1520  const auto *VD = cast<VarDecl>(DRE->getDecl());
1521  // Override only those variables that are really emitted already.
1522  if (LocalDeclMap.count(VD)) {
1523  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1524  return CreateMemTemp(DRE->getType(), VD->getName());
1525  });
1526  }
1527  }
1528  }
1529 }
1530 
1531 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1532  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1533  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1534  if (!CGF.HaveInsertPoint())
1535  return;
1536  {
1537  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1538  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1539  (void)PreCondScope.Privatize();
1540  // Get initial values of real counters.
1541  for (const Expr *I : S.inits()) {
1542  CGF.EmitIgnoredExpr(I);
1543  }
1544  }
1545  // Check that loop is executed at least one time.
1546  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1547 }
1548 
1550  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1551  if (!HaveInsertPoint())
1552  return;
1555  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1556  for (const Expr *C : LoopDirective->counters()) {
1557  SIMDLCVs.insert(
1558  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1559  }
1560  }
1561  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1562  auto CurPrivate = C->privates().begin();
1563  for (const Expr *E : C->varlists()) {
1564  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1565  const auto *PrivateVD =
1566  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1567  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1568  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1569  // Emit private VarDecl with copy init.
1570  EmitVarDecl(*PrivateVD);
1571  return GetAddrOfLocalVar(PrivateVD);
1572  });
1573  assert(IsRegistered && "linear var already registered as private");
1574  // Silence the warning about unused variable.
1575  (void)IsRegistered;
1576  } else {
1577  EmitVarDecl(*PrivateVD);
1578  }
1579  ++CurPrivate;
1580  }
1581  }
1582 }
1583 
1585  const OMPExecutableDirective &D,
1586  bool IsMonotonic) {
1587  if (!CGF.HaveInsertPoint())
1588  return;
1589  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1590  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1591  /*ignoreResult=*/true);
1592  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1593  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1594  // In presence of finite 'safelen', it may be unsafe to mark all
1595  // the memory instructions parallel, because loop-carried
1596  // dependences of 'safelen' iterations are possible.
1597  if (!IsMonotonic)
1599  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1600  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1601  /*ignoreResult=*/true);
1602  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1603  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1604  // In presence of finite 'safelen', it may be unsafe to mark all
1605  // the memory instructions parallel, because loop-carried
1606  // dependences of 'safelen' iterations are possible.
1607  CGF.LoopStack.setParallel(/*Enable=*/false);
1608  }
1609 }
1610 
1612  bool IsMonotonic) {
1613  // Walk clauses and process safelen/lastprivate.
1614  LoopStack.setParallel(!IsMonotonic);
1615  LoopStack.setVectorizeEnable();
1616  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1617 }
1618 
1620  const OMPLoopDirective &D,
1621  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1622  if (!HaveInsertPoint())
1623  return;
1624  llvm::BasicBlock *DoneBB = nullptr;
1625  auto IC = D.counters().begin();
1626  auto IPC = D.private_counters().begin();
1627  for (const Expr *F : D.finals()) {
1628  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1629  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1630  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1631  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1632  OrigVD->hasGlobalStorage() || CED) {
1633  if (!DoneBB) {
1634  if (llvm::Value *Cond = CondGen(*this)) {
1635  // If the first post-update expression is found, emit conditional
1636  // block if it was requested.
1637  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1638  DoneBB = createBasicBlock(".omp.final.done");
1639  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1640  EmitBlock(ThenBB);
1641  }
1642  }
1643  Address OrigAddr = Address::invalid();
1644  if (CED) {
1645  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1646  } else {
1647  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1648  /*RefersToEnclosingVariableOrCapture=*/false,
1649  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1650  OrigAddr = EmitLValue(&DRE).getAddress();
1651  }
1652  OMPPrivateScope VarScope(*this);
1653  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1654  (void)VarScope.Privatize();
1655  EmitIgnoredExpr(F);
1656  }
1657  ++IC;
1658  ++IPC;
1659  }
1660  if (DoneBB)
1661  EmitBlock(DoneBB, /*IsFinished=*/true);
1662 }
1663 
1665  const OMPLoopDirective &S,
1667  CGF.EmitOMPLoopBody(S, LoopExit);
1668  CGF.EmitStopPoint(&S);
1669 }
1670 
1671 /// Emit a helper variable and return corresponding lvalue.
1673  const DeclRefExpr *Helper) {
1674  auto VDecl = cast<VarDecl>(Helper->getDecl());
1675  CGF.EmitVarDecl(*VDecl);
1676  return CGF.EmitLValue(Helper);
1677 }
1678 
1680  PrePostActionTy &Action) {
1681  Action.Enter(CGF);
1683  "Expected simd directive");
1684  OMPLoopScope PreInitScope(CGF, S);
1685  // if (PreCond) {
1686  // for (IV in 0..LastIteration) BODY;
1687  // <Final counter/linear vars updates>;
1688  // }
1689  //
1693  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1694  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1695  }
1696 
1697  // Emit: if (PreCond) - begin.
1698  // If the condition constant folds and can be elided, avoid emitting the
1699  // whole loop.
1700  bool CondConstant;
1701  llvm::BasicBlock *ContBlock = nullptr;
1702  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1703  if (!CondConstant)
1704  return;
1705  } else {
1706  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1707  ContBlock = CGF.createBasicBlock("simd.if.end");
1708  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1709  CGF.getProfileCount(&S));
1710  CGF.EmitBlock(ThenBlock);
1711  CGF.incrementProfileCounter(&S);
1712  }
1713 
1714  // Emit the loop iteration variable.
1715  const Expr *IVExpr = S.getIterationVariable();
1716  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1717  CGF.EmitVarDecl(*IVDecl);
1718  CGF.EmitIgnoredExpr(S.getInit());
1719 
1720  // Emit the iterations count variable.
1721  // If it is not a variable, Sema decided to calculate iterations count on
1722  // each iteration (e.g., it is foldable into a constant).
1723  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1724  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1725  // Emit calculation of the iterations count.
1727  }
1728 
1729  CGF.EmitOMPSimdInit(S);
1730 
1731  emitAlignedClause(CGF, S);
1732  (void)CGF.EmitOMPLinearClauseInit(S);
1733  {
1734  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1735  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1736  CGF.EmitOMPLinearClause(S, LoopScope);
1737  CGF.EmitOMPPrivateClause(S, LoopScope);
1738  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1739  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1740  (void)LoopScope.Privatize();
1741  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1742  S.getInc(),
1743  [&S](CodeGenFunction &CGF) {
1745  CGF.EmitStopPoint(&S);
1746  },
1747  [](CodeGenFunction &) {});
1748  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1749  // Emit final copy of the lastprivate variables at the end of loops.
1750  if (HasLastprivateClause)
1751  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1752  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1754  [](CodeGenFunction &) { return nullptr; });
1755  }
1756  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1757  // Emit: if (PreCond) - end.
1758  if (ContBlock) {
1759  CGF.EmitBranch(ContBlock);
1760  CGF.EmitBlock(ContBlock, true);
1761  }
1762 }
1763 
1765  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1766  emitOMPSimdRegion(CGF, S, Action);
1767  };
1768  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1769  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1770 }
1771 
1772 void CodeGenFunction::EmitOMPOuterLoop(
1773  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1775  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1776  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1777  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1778  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1779 
1780  const Expr *IVExpr = S.getIterationVariable();
1781  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1782  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1783 
1784  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1785 
1786  // Start the loop with a block that tests the condition.
1787  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1788  EmitBlock(CondBlock);
1789  const SourceRange R = S.getSourceRange();
1790  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1791  SourceLocToDebugLoc(R.getEnd()));
1792 
1793  llvm::Value *BoolCondVal = nullptr;
1794  if (!DynamicOrOrdered) {
1795  // UB = min(UB, GlobalUB) or
1796  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1797  // 'distribute parallel for')
1798  EmitIgnoredExpr(LoopArgs.EUB);
1799  // IV = LB
1800  EmitIgnoredExpr(LoopArgs.Init);
1801  // IV < UB
1802  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1803  } else {
1804  BoolCondVal =
1805  RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1806  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1807  }
1808 
1809  // If there are any cleanups between here and the loop-exit scope,
1810  // create a block to stage a loop exit along.
1811  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1812  if (LoopScope.requiresCleanups())
1813  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1814 
1815  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1816  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1817  if (ExitBlock != LoopExit.getBlock()) {
1818  EmitBlock(ExitBlock);
1819  EmitBranchThroughCleanup(LoopExit);
1820  }
1821  EmitBlock(LoopBody);
1822 
1823  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1824  // LB for loop condition and emitted it above).
1825  if (DynamicOrOrdered)
1826  EmitIgnoredExpr(LoopArgs.Init);
1827 
1828  // Create a block for the increment.
1829  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1830  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1831 
1832  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1833  // with dynamic/guided scheduling and without ordered clause.
1835  LoopStack.setParallel(!IsMonotonic);
1836  else
1837  EmitOMPSimdInit(S, IsMonotonic);
1838 
1839  SourceLocation Loc = S.getLocStart();
1840 
1841  // when 'distribute' is not combined with a 'for':
1842  // while (idx <= UB) { BODY; ++idx; }
1843  // when 'distribute' is combined with a 'for'
1844  // (e.g. 'distribute parallel for')
1845  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1846  EmitOMPInnerLoop(
1847  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1848  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1849  CodeGenLoop(CGF, S, LoopExit);
1850  },
1851  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1852  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1853  });
1854 
1855  EmitBlock(Continue.getBlock());
1856  BreakContinueStack.pop_back();
1857  if (!DynamicOrOrdered) {
1858  // Emit "LB = LB + Stride", "UB = UB + Stride".
1859  EmitIgnoredExpr(LoopArgs.NextLB);
1860  EmitIgnoredExpr(LoopArgs.NextUB);
1861  }
1862 
1863  EmitBranch(CondBlock);
1864  LoopStack.pop();
1865  // Emit the fall-through block.
1866  EmitBlock(LoopExit.getBlock());
1867 
1868  // Tell the runtime we are done.
1869  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1870  if (!DynamicOrOrdered)
1871  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
1872  S.getDirectiveKind());
1873  };
1874  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1875 }
1876 
1877 void CodeGenFunction::EmitOMPForOuterLoop(
1878  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1879  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1880  const OMPLoopArguments &LoopArgs,
1881  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1882  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1883 
1884  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1885  const bool DynamicOrOrdered =
1886  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1887 
1888  assert((Ordered ||
1889  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1890  LoopArgs.Chunk != nullptr)) &&
1891  "static non-chunked schedule does not need outer loop");
1892 
1893  // Emit outer loop.
1894  //
1895  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1896  // When schedule(dynamic,chunk_size) is specified, the iterations are
1897  // distributed to threads in the team in chunks as the threads request them.
1898  // Each thread executes a chunk of iterations, then requests another chunk,
1899  // until no chunks remain to be distributed. Each chunk contains chunk_size
1900  // iterations, except for the last chunk to be distributed, which may have
1901  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1902  //
1903  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1904  // to threads in the team in chunks as the executing threads request them.
1905  // Each thread executes a chunk of iterations, then requests another chunk,
1906  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1907  // each chunk is proportional to the number of unassigned iterations divided
1908  // by the number of threads in the team, decreasing to 1. For a chunk_size
1909  // with value k (greater than 1), the size of each chunk is determined in the
1910  // same way, with the restriction that the chunks do not contain fewer than k
1911  // iterations (except for the last chunk to be assigned, which may have fewer
1912  // than k iterations).
1913  //
1914  // When schedule(auto) is specified, the decision regarding scheduling is
1915  // delegated to the compiler and/or runtime system. The programmer gives the
1916  // implementation the freedom to choose any possible mapping of iterations to
1917  // threads in the team.
1918  //
1919  // When schedule(runtime) is specified, the decision regarding scheduling is
1920  // deferred until run time, and the schedule and chunk size are taken from the
1921  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1922  // implementation defined
1923  //
1924  // while(__kmpc_dispatch_next(&LB, &UB)) {
1925  // idx = LB;
1926  // while (idx <= UB) { BODY; ++idx;
1927  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1928  // } // inner loop
1929  // }
1930  //
1931  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1932  // When schedule(static, chunk_size) is specified, iterations are divided into
1933  // chunks of size chunk_size, and the chunks are assigned to the threads in
1934  // the team in a round-robin fashion in the order of the thread number.
1935  //
1936  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1937  // while (idx <= UB) { BODY; ++idx; } // inner loop
1938  // LB = LB + ST;
1939  // UB = UB + ST;
1940  // }
1941  //
1942 
1943  const Expr *IVExpr = S.getIterationVariable();
1944  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1945  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1946 
1947  if (DynamicOrOrdered) {
1948  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1949  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1950  llvm::Value *LBVal = DispatchBounds.first;
1951  llvm::Value *UBVal = DispatchBounds.second;
1952  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1953  LoopArgs.Chunk};
1954  RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize,
1955  IVSigned, Ordered, DipatchRTInputValues);
1956  } else {
1957  CGOpenMPRuntime::StaticRTInput StaticInit(
1958  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1959  LoopArgs.ST, LoopArgs.Chunk);
1960  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
1961  ScheduleKind, StaticInit);
1962  }
1963 
1964  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1965  const unsigned IVSize,
1966  const bool IVSigned) {
1967  if (Ordered) {
1968  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1969  IVSigned);
1970  }
1971  };
1972 
1973  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1974  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1975  OuterLoopArgs.IncExpr = S.getInc();
1976  OuterLoopArgs.Init = S.getInit();
1977  OuterLoopArgs.Cond = S.getCond();
1978  OuterLoopArgs.NextLB = S.getNextLowerBound();
1979  OuterLoopArgs.NextUB = S.getNextUpperBound();
1980  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1981  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1982 }
1983 
1985  const unsigned IVSize, const bool IVSigned) {}
1986 
1987 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1988  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1989  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1990  const CodeGenLoopTy &CodeGenLoopContent) {
1991 
1992  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1993 
1994  // Emit outer loop.
1995  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1996  // dynamic
1997  //
1998 
1999  const Expr *IVExpr = S.getIterationVariable();
2000  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2001  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2002 
2003  CGOpenMPRuntime::StaticRTInput StaticInit(
2004  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2005  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2006  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit);
2007 
2008  // for combined 'distribute' and 'for' the increment expression of distribute
2009  // is store in DistInc. For 'distribute' alone, it is in Inc.
2010  Expr *IncExpr;
2012  IncExpr = S.getDistInc();
2013  else
2014  IncExpr = S.getInc();
2015 
2016  // this routine is shared by 'omp distribute parallel for' and
2017  // 'omp distribute': select the right EUB expression depending on the
2018  // directive
2019  OMPLoopArguments OuterLoopArgs;
2020  OuterLoopArgs.LB = LoopArgs.LB;
2021  OuterLoopArgs.UB = LoopArgs.UB;
2022  OuterLoopArgs.ST = LoopArgs.ST;
2023  OuterLoopArgs.IL = LoopArgs.IL;
2024  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2025  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2027  : S.getEnsureUpperBound();
2028  OuterLoopArgs.IncExpr = IncExpr;
2029  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2030  ? S.getCombinedInit()
2031  : S.getInit();
2032  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2033  ? S.getCombinedCond()
2034  : S.getCond();
2035  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2037  : S.getNextLowerBound();
2038  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2040  : S.getNextUpperBound();
2041 
2042  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2043  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2045 }
2046 
2047 static std::pair<LValue, LValue>
2049  const OMPExecutableDirective &S) {
2050  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2051  LValue LB =
2052  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2053  LValue UB =
2054  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2055 
2056  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2057  // parallel for') we need to use the 'distribute'
2058  // chunk lower and upper bounds rather than the whole loop iteration
2059  // space. These are parameters to the outlined function for 'parallel'
2060  // and we copy the bounds of the previous schedule into the
2061  // the current ones.
2062  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2063  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2064  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2065  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2066  PrevLBVal = CGF.EmitScalarConversion(
2067  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2068  LS.getIterationVariable()->getType(),
2070  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2071  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2072  PrevUBVal = CGF.EmitScalarConversion(
2073  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2074  LS.getIterationVariable()->getType(),
2076 
2077  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2078  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2079 
2080  return {LB, UB};
2081 }
2082 
2083 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2084 /// we need to use the LB and UB expressions generated by the worksharing
2085 /// code generation support, whereas in non combined situations we would
2086 /// just emit 0 and the LastIteration expression
2087 /// This function is necessary due to the difference of the LB and UB
2088 /// types for the RT emission routines for 'for_static_init' and
2089 /// 'for_dispatch_init'
2090 static std::pair<llvm::Value *, llvm::Value *>
2092  const OMPExecutableDirective &S,
2093  Address LB, Address UB) {
2094  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2095  const Expr *IVExpr = LS.getIterationVariable();
2096  // when implementing a dynamic schedule for a 'for' combined with a
2097  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2098  // is not normalized as each team only executes its own assigned
2099  // distribute chunk
2100  QualType IteratorTy = IVExpr->getType();
2101  llvm::Value *LBVal =
2102  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getLocStart());
2103  llvm::Value *UBVal =
2104  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getLocStart());
2105  return {LBVal, UBVal};
2106 }
2107 
2109  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2110  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2111  const auto &Dir = cast<OMPLoopDirective>(S);
2112  LValue LB =
2113  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2114  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2115  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2116  CapturedVars.push_back(LBCast);
2117  LValue UB =
2118  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2119 
2120  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2121  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2122  CapturedVars.push_back(UBCast);
2123 }
2124 
2125 static void
2127  const OMPLoopDirective &S,
2129  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2130  PrePostActionTy &Action) {
2131  Action.Enter(CGF);
2132  bool HasCancel = false;
2134  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2135  HasCancel = D->hasCancel();
2136  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2137  HasCancel = D->hasCancel();
2138  else if (const auto *D =
2139  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2140  HasCancel = D->hasCancel();
2141  }
2143  HasCancel);
2147  };
2148 
2150  CGF, S,
2151  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2152  CGInlinedWorksharingLoop,
2154 }
2155 
2158  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2160  S.getDistInc());
2161  };
2162  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2163  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2164 }
2165 
2168  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2170  S.getDistInc());
2171  };
2172  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2173  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2174 }
2175 
2177  const OMPDistributeSimdDirective &S) {
2178  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2180  };
2181  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2182  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2183 }
2184 
2186  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2187  // Emit SPMD target parallel for region as a standalone region.
2188  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2189  emitOMPSimdRegion(CGF, S, Action);
2190  };
2191  llvm::Function *Fn;
2192  llvm::Constant *Addr;
2193  // Emit target region as a standalone region.
2194  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2195  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2196  assert(Fn && Addr && "Target device function emission failed.");
2197 }
2198 
2200  const OMPTargetSimdDirective &S) {
2201  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2202  emitOMPSimdRegion(CGF, S, Action);
2203  };
2204  emitCommonOMPTargetDirective(*this, S, CodeGen);
2205 }
2206 
2207 namespace {
2208  struct ScheduleKindModifiersTy {
2212  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2215  : Kind(Kind), M1(M1), M2(M2) {}
2216  };
2217 } // namespace
2218 
2220  const OMPLoopDirective &S, Expr *EUB,
2221  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2222  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2223  // Emit the loop iteration variable.
2224  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2225  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2226  EmitVarDecl(*IVDecl);
2227 
2228  // Emit the iterations count variable.
2229  // If it is not a variable, Sema decided to calculate iterations count on each
2230  // iteration (e.g., it is foldable into a constant).
2231  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2232  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2233  // Emit calculation of the iterations count.
2234  EmitIgnoredExpr(S.getCalcLastIteration());
2235  }
2236 
2237  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2238 
2239  bool HasLastprivateClause;
2240  // Check pre-condition.
2241  {
2242  OMPLoopScope PreInitScope(*this, S);
2243  // Skip the entire loop if we don't meet the precondition.
2244  // If the condition constant folds and can be elided, avoid emitting the
2245  // whole loop.
2246  bool CondConstant;
2247  llvm::BasicBlock *ContBlock = nullptr;
2248  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2249  if (!CondConstant)
2250  return false;
2251  } else {
2252  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2253  ContBlock = createBasicBlock("omp.precond.end");
2254  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2255  getProfileCount(&S));
2256  EmitBlock(ThenBlock);
2257  incrementProfileCounter(&S);
2258  }
2259 
2260  RunCleanupsScope DoacrossCleanupScope(*this);
2261  bool Ordered = false;
2262  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2263  if (OrderedClause->getNumForLoops())
2264  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2265  else
2266  Ordered = true;
2267  }
2268 
2269  llvm::DenseSet<const Expr *> EmittedFinals;
2270  emitAlignedClause(*this, S);
2271  bool HasLinears = EmitOMPLinearClauseInit(S);
2272  // Emit helper vars inits.
2273 
2274  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2275  LValue LB = Bounds.first;
2276  LValue UB = Bounds.second;
2277  LValue ST =
2278  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2279  LValue IL =
2280  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2281 
2282  // Emit 'then' code.
2283  {
2284  OMPPrivateScope LoopScope(*this);
2285  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2286  // Emit implicit barrier to synchronize threads and avoid data races on
2287  // initialization of firstprivate variables and post-update of
2288  // lastprivate variables.
2289  CGM.getOpenMPRuntime().emitBarrierCall(
2290  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2291  /*ForceSimpleCall=*/true);
2292  }
2293  EmitOMPPrivateClause(S, LoopScope);
2294  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2295  EmitOMPReductionClauseInit(S, LoopScope);
2296  EmitOMPPrivateLoopCounters(S, LoopScope);
2297  EmitOMPLinearClause(S, LoopScope);
2298  (void)LoopScope.Privatize();
2299 
2300  // Detect the loop schedule kind and chunk.
2301  llvm::Value *Chunk = nullptr;
2302  OpenMPScheduleTy ScheduleKind;
2303  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2304  ScheduleKind.Schedule = C->getScheduleKind();
2305  ScheduleKind.M1 = C->getFirstScheduleModifier();
2306  ScheduleKind.M2 = C->getSecondScheduleModifier();
2307  if (const Expr *Ch = C->getChunkSize()) {
2308  Chunk = EmitScalarExpr(Ch);
2309  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2311  S.getLocStart());
2312  }
2313  }
2314  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2315  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2316  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2317  // If the static schedule kind is specified or if the ordered clause is
2318  // specified, and if no monotonic modifier is specified, the effect will
2319  // be as if the monotonic modifier was specified.
2320  if (RT.isStaticNonchunked(ScheduleKind.Schedule,
2321  /* Chunked */ Chunk != nullptr) &&
2322  !Ordered) {
2324  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2325  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2326  // When no chunk_size is specified, the iteration space is divided into
2327  // chunks that are approximately equal in size, and at most one chunk is
2328  // distributed to each thread. Note that the size of the chunks is
2329  // unspecified in this case.
2330  CGOpenMPRuntime::StaticRTInput StaticInit(
2331  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2332  UB.getAddress(), ST.getAddress());
2333  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
2334  ScheduleKind, StaticInit);
2335  JumpDest LoopExit =
2336  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2337  // UB = min(UB, GlobalUB);
2338  EmitIgnoredExpr(S.getEnsureUpperBound());
2339  // IV = LB;
2340  EmitIgnoredExpr(S.getInit());
2341  // while (idx <= UB) { BODY; ++idx; }
2342  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2343  S.getInc(),
2344  [&S, LoopExit](CodeGenFunction &CGF) {
2345  CGF.EmitOMPLoopBody(S, LoopExit);
2346  CGF.EmitStopPoint(&S);
2347  },
2348  [](CodeGenFunction &) {});
2349  EmitBlock(LoopExit.getBlock());
2350  // Tell the runtime we are done.
2351  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2352  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2353  S.getDirectiveKind());
2354  };
2355  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2356  } else {
2357  const bool IsMonotonic =
2358  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2359  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2360  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2361  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2362  // Emit the outer loop, which requests its work chunk [LB..UB] from
2363  // runtime and runs the inner loop to process it.
2364  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2365  ST.getAddress(), IL.getAddress(),
2366  Chunk, EUB);
2367  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2368  LoopArguments, CGDispatchBounds);
2369  }
2371  EmitOMPSimdFinal(S,
2372  [IL, &S](CodeGenFunction &CGF) {
2373  return CGF.Builder.CreateIsNotNull(
2374  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2375  });
2376  }
2377  EmitOMPReductionClauseFinal(
2378  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2379  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2380  : /*Parallel only*/ OMPD_parallel);
2381  // Emit post-update of the reduction variables if IsLastIter != 0.
2383  *this, S, [IL, &S](CodeGenFunction &CGF) {
2384  return CGF.Builder.CreateIsNotNull(
2385  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2386  });
2387  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2388  if (HasLastprivateClause)
2389  EmitOMPLastprivateClauseFinal(
2391  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
2392  }
2393  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2394  return CGF.Builder.CreateIsNotNull(
2395  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2396  });
2397  DoacrossCleanupScope.ForceCleanup();
2398  // We're now done with the loop, so jump to the continuation block.
2399  if (ContBlock) {
2400  EmitBranch(ContBlock);
2401  EmitBlock(ContBlock, /*IsFinished=*/true);
2402  }
2403  }
2404  return HasLastprivateClause;
2405 }
2406 
2407 /// The following two functions generate expressions for the loop lower
2408 /// and upper bounds in case of static and dynamic (dispatch) schedule
2409 /// of the associated 'for' or 'distribute' loop.
2410 static std::pair<LValue, LValue>
2412  const auto &LS = cast<OMPLoopDirective>(S);
2413  LValue LB =
2414  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2415  LValue UB =
2416  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2417  return {LB, UB};
2418 }
2419 
2420 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2421 /// consider the lower and upper bound expressions generated by the
2422 /// worksharing loop support, but we use 0 and the iteration space size as
2423 /// constants
2424 static std::pair<llvm::Value *, llvm::Value *>
2426  Address LB, Address UB) {
2427  const auto &LS = cast<OMPLoopDirective>(S);
2428  const Expr *IVExpr = LS.getIterationVariable();
2429  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2430  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2431  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2432  return {LBVal, UBVal};
2433 }
2434 
2436  bool HasLastprivates = false;
2437  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2438  PrePostActionTy &) {
2439  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2440  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2443  };
2444  {
2445  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2446  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2447  S.hasCancel());
2448  }
2449 
2450  // Emit an implicit barrier at the end.
2451  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2452  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2453 }
2454 
2456  bool HasLastprivates = false;
2457  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2458  PrePostActionTy &) {
2459  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2462  };
2463  {
2464  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2465  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2466  }
2467 
2468  // Emit an implicit barrier at the end.
2469  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2470  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2471 }
2472 
2474  const Twine &Name,
2475  llvm::Value *Init = nullptr) {
2476  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2477  if (Init)
2478  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2479  return LVal;
2480 }
2481 
2482 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2484  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2485  bool HasLastprivates = false;
2486  auto &&CodeGen = [&S, CapturedStmt, CS,
2487  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2488  ASTContext &C = CGF.getContext();
2489  QualType KmpInt32Ty =
2490  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2491  // Emit helper vars inits.
2492  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2493  CGF.Builder.getInt32(0));
2494  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2495  ? CGF.Builder.getInt32(CS->size() - 1)
2496  : CGF.Builder.getInt32(0);
2497  LValue UB =
2498  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2499  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2500  CGF.Builder.getInt32(1));
2501  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2502  CGF.Builder.getInt32(0));
2503  // Loop counter.
2504  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2505  OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2506  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2507  OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2508  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2509  // Generate condition for loop.
2510  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2512  // Increment for loop counter.
2513  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2514  S.getLocStart(), true);
2515  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2516  // Iterate through all sections and emit a switch construct:
2517  // switch (IV) {
2518  // case 0:
2519  // <SectionStmt[0]>;
2520  // break;
2521  // ...
2522  // case <NumSection> - 1:
2523  // <SectionStmt[<NumSection> - 1]>;
2524  // break;
2525  // }
2526  // .omp.sections.exit:
2527  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2528  llvm::SwitchInst *SwitchStmt =
2529  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getLocStart()),
2530  ExitBB, CS == nullptr ? 1 : CS->size());
2531  if (CS) {
2532  unsigned CaseNumber = 0;
2533  for (const Stmt *SubStmt : CS->children()) {
2534  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2535  CGF.EmitBlock(CaseBB);
2536  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2537  CGF.EmitStmt(SubStmt);
2538  CGF.EmitBranch(ExitBB);
2539  ++CaseNumber;
2540  }
2541  } else {
2542  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2543  CGF.EmitBlock(CaseBB);
2544  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2545  CGF.EmitStmt(CapturedStmt);
2546  CGF.EmitBranch(ExitBB);
2547  }
2548  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2549  };
2550 
2551  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2552  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2553  // Emit implicit barrier to synchronize threads and avoid data races on
2554  // initialization of firstprivate variables and post-update of lastprivate
2555  // variables.
2556  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2557  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2558  /*ForceSimpleCall=*/true);
2559  }
2560  CGF.EmitOMPPrivateClause(S, LoopScope);
2561  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2562  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2563  (void)LoopScope.Privatize();
2564 
2565  // Emit static non-chunked loop.
2566  OpenMPScheduleTy ScheduleKind;
2567  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2568  CGOpenMPRuntime::StaticRTInput StaticInit(
2569  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2570  LB.getAddress(), UB.getAddress(), ST.getAddress());
2571  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2572  CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2573  // UB = min(UB, GlobalUB);
2574  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2575  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2576  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2577  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2578  // IV = LB;
2579  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2580  // while (idx <= UB) { BODY; ++idx; }
2581  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2582  [](CodeGenFunction &) {});
2583  // Tell the runtime we are done.
2584  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2585  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2586  S.getDirectiveKind());
2587  };
2588  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2589  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2590  // Emit post-update of the reduction variables if IsLastIter != 0.
2591  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2592  return CGF.Builder.CreateIsNotNull(
2593  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2594  });
2595 
2596  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2597  if (HasLastprivates)
2599  S, /*NoFinals=*/false,
2600  CGF.Builder.CreateIsNotNull(
2601  CGF.EmitLoadOfScalar(IL, S.getLocStart())));
2602  };
2603 
2604  bool HasCancel = false;
2605  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2606  HasCancel = OSD->hasCancel();
2607  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2608  HasCancel = OPSD->hasCancel();
2609  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2610  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2611  HasCancel);
2612  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2613  // clause. Otherwise the barrier will be generated by the codegen for the
2614  // directive.
2615  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2616  // Emit implicit barrier to synchronize threads and avoid data races on
2617  // initialization of firstprivate variables.
2618  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2619  OMPD_unknown);
2620  }
2621 }
2622 
2624  {
2625  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2626  EmitSections(S);
2627  }
2628  // Emit an implicit barrier at the end.
2629  if (!S.getSingleClause<OMPNowaitClause>()) {
2630  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2631  OMPD_sections);
2632  }
2633 }
2634 
2636  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2638  };
2639  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2640  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2641  S.hasCancel());
2642 }
2643 
2645  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2648  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2649  // Check if there are any 'copyprivate' clauses associated with this
2650  // 'single' construct.
2651  // Build a list of copyprivate variables along with helper expressions
2652  // (<source>, <destination>, <destination>=<source> expressions)
2653  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2654  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2655  DestExprs.append(C->destination_exprs().begin(),
2656  C->destination_exprs().end());
2657  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2658  AssignmentOps.append(C->assignment_ops().begin(),
2659  C->assignment_ops().end());
2660  }
2661  // Emit code for 'single' region along with 'copyprivate' clauses
2662  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2663  Action.Enter(CGF);
2664  OMPPrivateScope SingleScope(CGF);
2665  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2666  CGF.EmitOMPPrivateClause(S, SingleScope);
2667  (void)SingleScope.Privatize();
2669  };
2670  {
2671  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2672  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
2673  CopyprivateVars, DestExprs,
2674  SrcExprs, AssignmentOps);
2675  }
2676  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2677  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2678  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2679  CGM.getOpenMPRuntime().emitBarrierCall(
2680  *this, S.getLocStart(),
2681  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2682  }
2683 }
2684 
2686  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2687  Action.Enter(CGF);
2689  };
2690  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2691  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
2692 }
2693 
2695  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2696  Action.Enter(CGF);
2698  };
2699  const Expr *Hint = nullptr;
2700  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2701  Hint = HintClause->getHint();
2702  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2703  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2705  CodeGen, S.getLocStart(), Hint);
2706 }
2707 
2709  const OMPParallelForDirective &S) {
2710  // Emit directive as a combined directive that consists of two implicit
2711  // directives: 'parallel' with 'for' directive.
2712  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2713  Action.Enter(CGF);
2714  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2717  };
2718  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2720 }
2721 
2723  const OMPParallelForSimdDirective &S) {
2724  // Emit directive as a combined directive that consists of two implicit
2725  // directives: 'parallel' with 'for' directive.
2726  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2727  Action.Enter(CGF);
2730  };
2731  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2733 }
2734 
2736  const OMPParallelSectionsDirective &S) {
2737  // Emit directive as a combined directive that consists of two implicit
2738  // directives: 'parallel' with 'sections' directive.
2739  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2740  Action.Enter(CGF);
2741  CGF.EmitSections(S);
2742  };
2743  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2745 }
2746 
2748  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2749  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2750  OMPTaskDataTy &Data) {
2751  // Emit outlined function for task construct.
2752  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2753  auto I = CS->getCapturedDecl()->param_begin();
2754  auto PartId = std::next(I);
2755  auto TaskT = std::next(I, 4);
2756  // Check if the task is final
2757  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2758  // If the condition constant folds and can be elided, try to avoid emitting
2759  // the condition and the dead arm of the if/else.
2760  const Expr *Cond = Clause->getCondition();
2761  bool CondConstant;
2762  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2763  Data.Final.setInt(CondConstant);
2764  else
2765  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2766  } else {
2767  // By default the task is not final.
2768  Data.Final.setInt(/*IntVal=*/false);
2769  }
2770  // Check if the task has 'priority' clause.
2771  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2772  const Expr *Prio = Clause->getPriority();
2773  Data.Priority.setInt(/*IntVal=*/true);
2774  Data.Priority.setPointer(EmitScalarConversion(
2775  EmitScalarExpr(Prio), Prio->getType(),
2776  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2777  Prio->getExprLoc()));
2778  }
2779  // The first function argument for tasks is a thread id, the second one is a
2780  // part id (0 for tied tasks, >=0 for untied task).
2781  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2782  // Get list of private variables.
2783  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2784  auto IRef = C->varlist_begin();
2785  for (const Expr *IInit : C->private_copies()) {
2786  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2787  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2788  Data.PrivateVars.push_back(*IRef);
2789  Data.PrivateCopies.push_back(IInit);
2790  }
2791  ++IRef;
2792  }
2793  }
2794  EmittedAsPrivate.clear();
2795  // Get list of firstprivate variables.
2796  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2797  auto IRef = C->varlist_begin();
2798  auto IElemInitRef = C->inits().begin();
2799  for (const Expr *IInit : C->private_copies()) {
2800  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2801  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2802  Data.FirstprivateVars.push_back(*IRef);
2803  Data.FirstprivateCopies.push_back(IInit);
2804  Data.FirstprivateInits.push_back(*IElemInitRef);
2805  }
2806  ++IRef;
2807  ++IElemInitRef;
2808  }
2809  }
2810  // Get list of lastprivate variables (for taskloops).
2811  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2812  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2813  auto IRef = C->varlist_begin();
2814  auto ID = C->destination_exprs().begin();
2815  for (const Expr *IInit : C->private_copies()) {
2816  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2817  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2818  Data.LastprivateVars.push_back(*IRef);
2819  Data.LastprivateCopies.push_back(IInit);
2820  }
2821  LastprivateDstsOrigs.insert(
2822  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2823  cast<DeclRefExpr>(*IRef)});
2824  ++IRef;
2825  ++ID;
2826  }
2827  }
2830  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2831  auto IPriv = C->privates().begin();
2832  auto IRed = C->reduction_ops().begin();
2833  auto ILHS = C->lhs_exprs().begin();
2834  auto IRHS = C->rhs_exprs().begin();
2835  for (const Expr *Ref : C->varlists()) {
2836  Data.ReductionVars.emplace_back(Ref);
2837  Data.ReductionCopies.emplace_back(*IPriv);
2838  Data.ReductionOps.emplace_back(*IRed);
2839  LHSs.emplace_back(*ILHS);
2840  RHSs.emplace_back(*IRHS);
2841  std::advance(IPriv, 1);
2842  std::advance(IRed, 1);
2843  std::advance(ILHS, 1);
2844  std::advance(IRHS, 1);
2845  }
2846  }
2847  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2848  *this, S.getLocStart(), LHSs, RHSs, Data);
2849  // Build list of dependences.
2850  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2851  for (const Expr *IRef : C->varlists())
2852  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2853  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2854  CapturedRegion](CodeGenFunction &CGF,
2855  PrePostActionTy &Action) {
2856  // Set proper addresses for generated private copies.
2857  OMPPrivateScope Scope(CGF);
2858  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2859  !Data.LastprivateVars.empty()) {
2860  enum { PrivatesParam = 2, CopyFnParam = 3 };
2861  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2862  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2863  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2864  CS->getCapturedDecl()->getParam(PrivatesParam)));
2865  // Map privates.
2868  CallArgs.push_back(PrivatesPtr);
2869  for (const Expr *E : Data.PrivateVars) {
2870  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2871  Address PrivatePtr = CGF.CreateMemTemp(
2872  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2873  PrivatePtrs.emplace_back(VD, PrivatePtr);
2874  CallArgs.push_back(PrivatePtr.getPointer());
2875  }
2876  for (const Expr *E : Data.FirstprivateVars) {
2877  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2878  Address PrivatePtr =
2879  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2880  ".firstpriv.ptr.addr");
2881  PrivatePtrs.emplace_back(VD, PrivatePtr);
2882  CallArgs.push_back(PrivatePtr.getPointer());
2883  }
2884  for (const Expr *E : Data.LastprivateVars) {
2885  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2886  Address PrivatePtr =
2887  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2888  ".lastpriv.ptr.addr");
2889  PrivatePtrs.emplace_back(VD, PrivatePtr);
2890  CallArgs.push_back(PrivatePtr.getPointer());
2891  }
2892  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2893  CopyFn, CallArgs);
2894  for (const auto &Pair : LastprivateDstsOrigs) {
2895  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2896  DeclRefExpr DRE(
2897  const_cast<VarDecl *>(OrigVD),
2898  /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2899  OrigVD) != nullptr,
2900  Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2901  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2902  return CGF.EmitLValue(&DRE).getAddress();
2903  });
2904  }
2905  for (const auto &Pair : PrivatePtrs) {
2906  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2907  CGF.getContext().getDeclAlign(Pair.first));
2908  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2909  }
2910  }
2911  if (Data.Reductions) {
2912  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2914  Data.ReductionOps);
2915  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2916  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2917  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2918  RedCG.emitSharedLValue(CGF, Cnt);
2919  RedCG.emitAggregateType(CGF, Cnt);
2920  // FIXME: This must removed once the runtime library is fixed.
2921  // Emit required threadprivate variables for
2922  // initilizer/combiner/finalizer.
2923  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2924  RedCG, Cnt);
2925  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2926  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2927  Replacement =
2928  Address(CGF.EmitScalarConversion(
2929  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2930  CGF.getContext().getPointerType(
2931  Data.ReductionCopies[Cnt]->getType()),
2932  Data.ReductionCopies[Cnt]->getExprLoc()),
2933  Replacement.getAlignment());
2934  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2935  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2936  [Replacement]() { return Replacement; });
2937  }
2938  }
2939  // Privatize all private variables except for in_reduction items.
2940  (void)Scope.Privatize();
2941  SmallVector<const Expr *, 4> InRedVars;
2942  SmallVector<const Expr *, 4> InRedPrivs;
2944  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2945  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2946  auto IPriv = C->privates().begin();
2947  auto IRed = C->reduction_ops().begin();
2948  auto ITD = C->taskgroup_descriptors().begin();
2949  for (const Expr *Ref : C->varlists()) {
2950  InRedVars.emplace_back(Ref);
2951  InRedPrivs.emplace_back(*IPriv);
2952  InRedOps.emplace_back(*IRed);
2953  TaskgroupDescriptors.emplace_back(*ITD);
2954  std::advance(IPriv, 1);
2955  std::advance(IRed, 1);
2956  std::advance(ITD, 1);
2957  }
2958  }
2959  // Privatize in_reduction items here, because taskgroup descriptors must be
2960  // privatized earlier.
2961  OMPPrivateScope InRedScope(CGF);
2962  if (!InRedVars.empty()) {
2963  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
2964  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2965  RedCG.emitSharedLValue(CGF, Cnt);
2966  RedCG.emitAggregateType(CGF, Cnt);
2967  // The taskgroup descriptor variable is always implicit firstprivate and
2968  // privatized already during procoessing of the firstprivates.
2969  // FIXME: This must removed once the runtime library is fixed.
2970  // Emit required threadprivate variables for
2971  // initilizer/combiner/finalizer.
2972  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2973  RedCG, Cnt);
2974  llvm::Value *ReductionsPtr =
2975  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
2976  TaskgroupDescriptors[Cnt]->getExprLoc());
2977  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2978  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2979  Replacement = Address(
2980  CGF.EmitScalarConversion(
2981  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2982  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2983  InRedPrivs[Cnt]->getExprLoc()),
2984  Replacement.getAlignment());
2985  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2986  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
2987  [Replacement]() { return Replacement; });
2988  }
2989  }
2990  (void)InRedScope.Privatize();
2991 
2992  Action.Enter(CGF);
2993  BodyGen(CGF);
2994  };
2995  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2996  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2997  Data.NumberOfParts);
2998  OMPLexicalScope Scope(*this, S);
2999  TaskGen(*this, OutlinedFn, Data);
3000 }
3001 
3002 static ImplicitParamDecl *
3004  QualType Ty, CapturedDecl *CD,
3005  SourceLocation Loc) {
3006  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3008  auto *OrigRef = DeclRefExpr::Create(
3009  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3010  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3011  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3013  auto *PrivateRef = DeclRefExpr::Create(
3014  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3015  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3016  QualType ElemType = C.getBaseElementType(Ty);
3017  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3019  auto *InitRef = DeclRefExpr::Create(
3020  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3021  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3022  PrivateVD->setInitStyle(VarDecl::CInit);
3023  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3024  InitRef, /*BasePath=*/nullptr,
3025  VK_RValue));
3026  Data.FirstprivateVars.emplace_back(OrigRef);
3027  Data.FirstprivateCopies.emplace_back(PrivateRef);
3028  Data.FirstprivateInits.emplace_back(InitRef);
3029  return OrigVD;
3030 }
3031 
3033  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3034  OMPTargetDataInfo &InputInfo) {
3035  // Emit outlined function for task construct.
3036  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3037  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3038  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3039  auto I = CS->getCapturedDecl()->param_begin();
3040  auto PartId = std::next(I);
3041  auto TaskT = std::next(I, 4);
3042  OMPTaskDataTy Data;
3043  // The task is not final.
3044  Data.Final.setInt(/*IntVal=*/false);
3045  // Get list of firstprivate variables.
3046  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3047  auto IRef = C->varlist_begin();
3048  auto IElemInitRef = C->inits().begin();
3049  for (auto *IInit : C->private_copies()) {
3050  Data.FirstprivateVars.push_back(*IRef);
3051  Data.FirstprivateCopies.push_back(IInit);
3052  Data.FirstprivateInits.push_back(*IElemInitRef);
3053  ++IRef;
3054  ++IElemInitRef;
3055  }
3056  }
3057  OMPPrivateScope TargetScope(*this);
3058  VarDecl *BPVD = nullptr;
3059  VarDecl *PVD = nullptr;
3060  VarDecl *SVD = nullptr;
3061  if (InputInfo.NumberOfTargetItems > 0) {
3062  auto *CD = CapturedDecl::Create(
3063  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3064  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3065  QualType BaseAndPointersType = getContext().getConstantArrayType(
3066  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3067  /*IndexTypeQuals=*/0);
3069  getContext(), Data, BaseAndPointersType, CD, S.getLocStart());
3071  getContext(), Data, BaseAndPointersType, CD, S.getLocStart());
3072  QualType SizesType = getContext().getConstantArrayType(
3073  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3074  /*IndexTypeQuals=*/0);
3075  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3076  S.getLocStart());
3077  TargetScope.addPrivate(
3078  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3079  TargetScope.addPrivate(PVD,
3080  [&InputInfo]() { return InputInfo.PointersArray; });
3081  TargetScope.addPrivate(SVD,
3082  [&InputInfo]() { return InputInfo.SizesArray; });
3083  }
3084  (void)TargetScope.Privatize();
3085  // Build list of dependences.
3086  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3087  for (const Expr *IRef : C->varlists())
3088  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3089  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3090  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3091  // Set proper addresses for generated private copies.
3092  OMPPrivateScope Scope(CGF);
3093  if (!Data.FirstprivateVars.empty()) {
3094  enum { PrivatesParam = 2, CopyFnParam = 3 };
3095  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3096  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3097  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3098  CS->getCapturedDecl()->getParam(PrivatesParam)));
3099  // Map privates.
3102  CallArgs.push_back(PrivatesPtr);
3103  for (const Expr *E : Data.FirstprivateVars) {
3104  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3105  Address PrivatePtr =
3106  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3107  ".firstpriv.ptr.addr");
3108  PrivatePtrs.emplace_back(VD, PrivatePtr);
3109  CallArgs.push_back(PrivatePtr.getPointer());
3110  }
3111  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3112  CopyFn, CallArgs);
3113  for (const auto &Pair : PrivatePtrs) {
3114  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3115  CGF.getContext().getDeclAlign(Pair.first));
3116  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3117  }
3118  }
3119  // Privatize all private variables except for in_reduction items.
3120  (void)Scope.Privatize();
3121  if (InputInfo.NumberOfTargetItems > 0) {
3122  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3123  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3124  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3125  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3126  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3127  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3128  }
3129 
3130  Action.Enter(CGF);
3131  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3132  BodyGen(CGF);
3133  };
3134  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3135  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3136  Data.NumberOfParts);
3137  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3138  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3139  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3140  SourceLocation());
3141 
3142  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn,
3143  SharedsTy, CapturedStruct, &IfCond, Data);
3144 }
3145 
3147  // Emit outlined function for task construct.
3148  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3149  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3150  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3151  const Expr *IfCond = nullptr;
3152  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3153  if (C->getNameModifier() == OMPD_unknown ||
3154  C->getNameModifier() == OMPD_task) {
3155  IfCond = C->getCondition();
3156  break;
3157  }
3158  }
3159 
3160  OMPTaskDataTy Data;
3161  // Check if we should emit tied or untied task.
3162  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3163  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3164  CGF.EmitStmt(CS->getCapturedStmt());
3165  };
3166  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3167  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3168  const OMPTaskDataTy &Data) {
3169  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
3170  SharedsTy, CapturedStruct, IfCond,
3171  Data);
3172  };
3173  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3174 }
3175 
3177  const OMPTaskyieldDirective &S) {
3178  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
3179 }
3180 
3182  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
3183 }
3184 
3186  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
3187 }
3188 
3190  const OMPTaskgroupDirective &S) {
3191  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3192  Action.Enter(CGF);
3193  if (const Expr *E = S.getReductionRef()) {
3196  OMPTaskDataTy Data;
3197  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3198  auto IPriv = C->privates().begin();
3199  auto IRed = C->reduction_ops().begin();
3200  auto ILHS = C->lhs_exprs().begin();
3201  auto IRHS = C->rhs_exprs().begin();
3202  for (const Expr *Ref : C->varlists()) {
3203  Data.ReductionVars.emplace_back(Ref);
3204  Data.ReductionCopies.emplace_back(*IPriv);
3205  Data.ReductionOps.emplace_back(*IRed);
3206  LHSs.emplace_back(*ILHS);
3207  RHSs.emplace_back(*IRHS);
3208  std::advance(IPriv, 1);
3209  std::advance(IRed, 1);
3210  std::advance(ILHS, 1);
3211  std::advance(IRHS, 1);
3212  }
3213  }
3214  llvm::Value *ReductionDesc =
3215  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(),
3216  LHSs, RHSs, Data);
3217  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3218  CGF.EmitVarDecl(*VD);
3219  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3220  /*Volatile=*/false, E->getType());
3221  }
3223  };
3224  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3225  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
3226 }
3227 
3229  CGM.getOpenMPRuntime().emitFlush(
3230  *this,
3231  [&S]() -> ArrayRef<const Expr *> {
3232  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3233  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3234  FlushClause->varlist_end());
3235  return llvm::None;
3236  }(),
3237  S.getLocStart());
3238 }
3239 
3241  const CodeGenLoopTy &CodeGenLoop,
3242  Expr *IncExpr) {
3243  // Emit the loop iteration variable.
3244  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3245  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3246  EmitVarDecl(*IVDecl);
3247 
3248  // Emit the iterations count variable.
3249  // If it is not a variable, Sema decided to calculate iterations count on each
3250  // iteration (e.g., it is foldable into a constant).
3251  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3252  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3253  // Emit calculation of the iterations count.
3254  EmitIgnoredExpr(S.getCalcLastIteration());
3255  }
3256 
3257  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3258 
3259  bool HasLastprivateClause = false;
3260  // Check pre-condition.
3261  {
3262  OMPLoopScope PreInitScope(*this, S);
3263  // Skip the entire loop if we don't meet the precondition.
3264  // If the condition constant folds and can be elided, avoid emitting the
3265  // whole loop.
3266  bool CondConstant;
3267  llvm::BasicBlock *ContBlock = nullptr;
3268  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3269  if (!CondConstant)
3270  return;
3271  } else {
3272  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3273  ContBlock = createBasicBlock("omp.precond.end");
3274  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3275  getProfileCount(&S));
3276  EmitBlock(ThenBlock);
3277  incrementProfileCounter(&S);
3278  }
3279 
3280  emitAlignedClause(*this, S);
3281  // Emit 'then' code.
3282  {
3283  // Emit helper vars inits.
3284 
3285  LValue LB = EmitOMPHelperVar(
3286  *this, cast<DeclRefExpr>(
3289  : S.getLowerBoundVariable())));
3290  LValue UB = EmitOMPHelperVar(
3291  *this, cast<DeclRefExpr>(
3294  : S.getUpperBoundVariable())));
3295  LValue ST =
3296  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3297  LValue IL =
3298  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3299 
3300  OMPPrivateScope LoopScope(*this);
3301  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3302  // Emit implicit barrier to synchronize threads and avoid data races
3303  // on initialization of firstprivate variables and post-update of
3304  // lastprivate variables.
3305  CGM.getOpenMPRuntime().emitBarrierCall(
3306  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
3307  /*ForceSimpleCall=*/true);
3308  }
3309  EmitOMPPrivateClause(S, LoopScope);
3313  EmitOMPReductionClauseInit(S, LoopScope);
3314  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3315  EmitOMPPrivateLoopCounters(S, LoopScope);
3316  (void)LoopScope.Privatize();
3317 
3318  // Detect the distribute schedule kind and chunk.
3319  llvm::Value *Chunk = nullptr;
3321  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3322  ScheduleKind = C->getDistScheduleKind();
3323  if (const Expr *Ch = C->getChunkSize()) {
3324  Chunk = EmitScalarExpr(Ch);
3325  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3327  S.getLocStart());
3328  }
3329  }
3330  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3331  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3332 
3333  // OpenMP [2.10.8, distribute Construct, Description]
3334  // If dist_schedule is specified, kind must be static. If specified,
3335  // iterations are divided into chunks of size chunk_size, chunks are
3336  // assigned to the teams of the league in a round-robin fashion in the
3337  // order of the team number. When no chunk_size is specified, the
3338  // iteration space is divided into chunks that are approximately equal
3339  // in size, and at most one chunk is distributed to each team of the
3340  // league. The size of the chunks is unspecified in this case.
3341  if (RT.isStaticNonchunked(ScheduleKind,
3342  /* Chunked */ Chunk != nullptr)) {
3344  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3345  CGOpenMPRuntime::StaticRTInput StaticInit(
3346  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3347  LB.getAddress(), UB.getAddress(), ST.getAddress());
3348  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind,
3349  StaticInit);
3350  JumpDest LoopExit =
3351  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3352  // UB = min(UB, GlobalUB);
3355  : S.getEnsureUpperBound());
3356  // IV = LB;
3358  ? S.getCombinedInit()
3359  : S.getInit());
3360 
3361  const Expr *Cond =
3363  ? S.getCombinedCond()
3364  : S.getCond();
3365 
3366  // for distribute alone, codegen
3367  // while (idx <= UB) { BODY; ++idx; }
3368  // when combined with 'for' (e.g. as in 'distribute parallel for')
3369  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3370  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3371  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3372  CodeGenLoop(CGF, S, LoopExit);
3373  },
3374  [](CodeGenFunction &) {});
3375  EmitBlock(LoopExit.getBlock());
3376  // Tell the runtime we are done.
3377  RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind());
3378  } else {
3379  // Emit the outer loop, which requests its work chunk [LB..UB] from
3380  // runtime and runs the inner loop to process it.
3381  const OMPLoopArguments LoopArguments = {
3382  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3383  Chunk};
3384  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3385  CodeGenLoop);
3386  }
3388  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3389  return CGF.Builder.CreateIsNotNull(
3390  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3391  });
3392  }
3396  OpenMPDirectiveKind ReductionKind = OMPD_unknown;
3399  ReductionKind = OMPD_parallel_for_simd;
3400  } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3401  ReductionKind = OMPD_parallel_for;
3402  } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3403  ReductionKind = OMPD_simd;
3404  } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
3406  llvm_unreachable(
3407  "No reduction clauses is allowed in distribute directive.");
3408  }
3409  EmitOMPReductionClauseFinal(S, ReductionKind);
3410  // Emit post-update of the reduction variables if IsLastIter != 0.
3412  *this, S, [IL, &S](CodeGenFunction &CGF) {
3413  return CGF.Builder.CreateIsNotNull(
3414  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3415  });
3416  }
3417  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3418  if (HasLastprivateClause) {
3419  EmitOMPLastprivateClauseFinal(
3420  S, /*NoFinals=*/false,
3421  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
3422  }
3423  }
3424 
3425  // We're now done with the loop, so jump to the continuation block.
3426  if (ContBlock) {
3427  EmitBranch(ContBlock);
3428  EmitBlock(ContBlock, true);
3429  }
3430  }
3431 }
3432 
3434  const OMPDistributeDirective &S) {
3435  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3437  };
3438  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3439  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3440 }
3441 
3442 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3443  const CapturedStmt *S) {
3444  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3446  CGF.CapturedStmtInfo = &CapStmtInfo;
3447  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3448  Fn->setDoesNotRecurse();
3449  return Fn;
3450 }
3451 
3453  if (S.hasClausesOfKind<OMPDependClause>()) {
3454  assert(!S.getAssociatedStmt() &&
3455  "No associated statement must be in ordered depend construct.");
3456  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3457  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3458  return;
3459  }
3460  const auto *C = S.getSingleClause<OMPSIMDClause>();
3461  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3462  PrePostActionTy &Action) {
3463  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3464  if (C) {
3466  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3467  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3468  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3469  OutlinedFn, CapturedVars);
3470  } else {
3471  Action.Enter(CGF);
3472  CGF.EmitStmt(CS->getCapturedStmt());
3473  }
3474  };
3475  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3476  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
3477 }
3478 
3480  QualType SrcType, QualType DestType,
3481  SourceLocation Loc) {
3482  assert(CGF.hasScalarEvaluationKind(DestType) &&
3483  "DestType must have scalar evaluation kind.");
3484  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3485  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3486  DestType, Loc)
3488  Val.getComplexVal(), SrcType, DestType, Loc);
3489 }
3490 
3493  QualType DestType, SourceLocation Loc) {
3494  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3495  "DestType must have complex evaluation kind.");
3496  CodeGenFunction::ComplexPairTy ComplexVal;
3497  if (Val.isScalar()) {
3498  // Convert the input element to the element type of the complex.
3499  QualType DestElementType =
3500  DestType->castAs<ComplexType>()->getElementType();
3501  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3502  Val.getScalarVal(), SrcType, DestElementType, Loc);
3503  ComplexVal = CodeGenFunction::ComplexPairTy(
3504  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3505  } else {
3506  assert(Val.isComplex() && "Must be a scalar or complex.");
3507  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3508  QualType DestElementType =
3509  DestType->castAs<ComplexType>()->getElementType();
3510  ComplexVal.first = CGF.EmitScalarConversion(
3511  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3512  ComplexVal.second = CGF.EmitScalarConversion(
3513  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3514  }
3515  return ComplexVal;
3516 }
3517 
3518 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3519  LValue LVal, RValue RVal) {
3520  if (LVal.isGlobalReg()) {
3521  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3522  } else {
3523  CGF.EmitAtomicStore(RVal, LVal,
3524  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3525  : llvm::AtomicOrdering::Monotonic,
3526  LVal.isVolatile(), /*IsInit=*/false);
3527  }
3528 }
3529 
3531  QualType RValTy, SourceLocation Loc) {
3532  switch (getEvaluationKind(LVal.getType())) {
3533  case TEK_Scalar:
3534  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3535  *this, RVal, RValTy, LVal.getType(), Loc)),
3536  LVal);
3537  break;
3538  case TEK_Complex:
3539  EmitStoreOfComplex(
3540  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3541  /*isInit=*/false);
3542  break;
3543  case TEK_Aggregate:
3544  llvm_unreachable("Must be a scalar or complex.");
3545  }
3546 }
3547 
3548 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3549  const Expr *X, const Expr *V,
3550  SourceLocation Loc) {
3551  // v = x;
3552  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3553  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3554  LValue XLValue = CGF.EmitLValue(X);
3555  LValue VLValue = CGF.EmitLValue(V);
3556  RValue Res = XLValue.isGlobalReg()
3557  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3558  : CGF.EmitAtomicLoad(
3559  XLValue, Loc,
3560  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3561  : llvm::AtomicOrdering::Monotonic,
3562  XLValue.isVolatile());
3563  // OpenMP, 2.12.6, atomic Construct
3564  // Any atomic construct with a seq_cst clause forces the atomically
3565  // performed operation to include an implicit flush operation without a
3566  // list.
3567  if (IsSeqCst)
3568  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3569  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3570 }
3571 
3572 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3573  const Expr *X, const Expr *E,
3574  SourceLocation Loc) {
3575  // x = expr;
3576  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3577  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3578  // OpenMP, 2.12.6, atomic Construct
3579  // Any atomic construct with a seq_cst clause forces the atomically
3580  // performed operation to include an implicit flush operation without a
3581  // list.
3582  if (IsSeqCst)
3583  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3584 }
3585 
3586 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3587  RValue Update,
3588  BinaryOperatorKind BO,
3589  llvm::AtomicOrdering AO,
3590  bool IsXLHSInRHSPart) {
3591  ASTContext &Context = CGF.getContext();
3592  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3593  // expression is simple and atomic is allowed for the given type for the
3594  // target platform.
3595  if (BO == BO_Comma || !Update.isScalar() ||
3596  !Update.getScalarVal()->getType()->isIntegerTy() ||
3597  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3598  (Update.getScalarVal()->getType() !=
3599  X.getAddress().getElementType())) ||
3600  !X.getAddress().getElementType()->isIntegerTy() ||
3601  !Context.getTargetInfo().hasBuiltinAtomic(
3602  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3603  return std::make_pair(false, RValue::get(nullptr));
3604 
3605  llvm::AtomicRMWInst::BinOp RMWOp;
3606  switch (BO) {
3607  case BO_Add:
3608  RMWOp = llvm::AtomicRMWInst::Add;
3609  break;
3610  case BO_Sub:
3611  if (!IsXLHSInRHSPart)
3612  return std::make_pair(false, RValue::get(nullptr));
3613  RMWOp = llvm::AtomicRMWInst::Sub;
3614  break;
3615  case BO_And:
3616  RMWOp = llvm::AtomicRMWInst::And;
3617  break;
3618  case BO_Or:
3619  RMWOp = llvm::AtomicRMWInst::Or;
3620  break;
3621  case BO_Xor:
3622  RMWOp = llvm::AtomicRMWInst::Xor;
3623  break;
3624  case BO_LT:
3626  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3627  : llvm::AtomicRMWInst::Max)
3628  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3629  : llvm::AtomicRMWInst::UMax);
3630  break;
3631  case BO_GT:
3633  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3634  : llvm::AtomicRMWInst::Min)
3635  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3636  : llvm::AtomicRMWInst::UMin);
3637  break;
3638  case BO_Assign:
3639  RMWOp = llvm::AtomicRMWInst::Xchg;
3640  break;
3641  case BO_Mul:
3642  case BO_Div:
3643  case BO_Rem:
3644  case BO_Shl:
3645  case BO_Shr:
3646  case BO_LAnd:
3647  case BO_LOr:
3648  return std::make_pair(false, RValue::get(nullptr));
3649  case BO_PtrMemD:
3650  case BO_PtrMemI:
3651  case BO_LE:
3652  case BO_GE:
3653  case BO_EQ:
3654  case BO_NE:
3655  case BO_Cmp:
3656  case BO_AddAssign:
3657  case BO_SubAssign:
3658  case BO_AndAssign:
3659  case BO_OrAssign:
3660  case BO_XorAssign:
3661  case BO_MulAssign:
3662  case BO_DivAssign:
3663  case BO_RemAssign:
3664  case BO_ShlAssign:
3665  case BO_ShrAssign:
3666  case BO_Comma:
3667  llvm_unreachable("Unsupported atomic update operation");
3668  }
3669  llvm::Value *UpdateVal = Update.getScalarVal();
3670  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3671  UpdateVal = CGF.Builder.CreateIntCast(
3672  IC, X.getAddress().getElementType(),
3674  }
3675  llvm::Value *Res =
3676  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3677  return std::make_pair(true, RValue::get(Res));
3678 }
3679 
3681  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3682  llvm::AtomicOrdering AO, SourceLocation Loc,
3683  const llvm::function_ref<RValue(RValue)> CommonGen) {
3684  // Update expressions are allowed to have the following forms:
3685  // x binop= expr; -> xrval + expr;
3686  // x++, ++x -> xrval + 1;
3687  // x--, --x -> xrval - 1;
3688  // x = x binop expr; -> xrval binop expr
3689  // x = expr Op x; - > expr binop xrval;
3690  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3691  if (!Res.first) {
3692  if (X.isGlobalReg()) {
3693  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3694  // 'xrval'.
3695  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3696  } else {
3697  // Perform compare-and-swap procedure.
3698  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3699  }
3700  }
3701  return Res;
3702 }
3703 
3704 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3705  const Expr *X, const Expr *E,
3706  const Expr *UE, bool IsXLHSInRHSPart,
3707  SourceLocation Loc) {
3708  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3709  "Update expr in 'atomic update' must be a binary operator.");
3710  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3711  // Update expressions are allowed to have the following forms:
3712  // x binop= expr; -> xrval + expr;
3713  // x++, ++x -> xrval + 1;
3714  // x--, --x -> xrval - 1;
3715  // x = x binop expr; -> xrval binop expr
3716  // x = expr Op x; - > expr binop xrval;
3717  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3718  LValue XLValue = CGF.EmitLValue(X);
3719  RValue ExprRValue = CGF.EmitAnyExpr(E);
3720  llvm::AtomicOrdering AO = IsSeqCst
3721  ? llvm::AtomicOrdering::SequentiallyConsistent
3722  : llvm::AtomicOrdering::Monotonic;
3723  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3724  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3725  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3726  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3727  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3728  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3729  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3730  return CGF.EmitAnyExpr(UE);
3731  };
3733  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3734  // OpenMP, 2.12.6, atomic Construct
3735  // Any atomic construct with a seq_cst clause forces the atomically
3736  // performed operation to include an implicit flush operation without a
3737  // list.
3738  if (IsSeqCst)
3739  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3740 }
3741 
3743  QualType SourceType, QualType ResType,
3744  SourceLocation Loc) {
3745  switch (CGF.getEvaluationKind(ResType)) {
3746  case TEK_Scalar:
3747  return RValue::get(
3748  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3749  case TEK_Complex: {
3750  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3751  return RValue::getComplex(Res.first, Res.second);
3752  }
3753  case TEK_Aggregate:
3754  break;
3755  }
3756  llvm_unreachable("Must be a scalar or complex.");
3757 }
3758 
3759 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3760  bool IsPostfixUpdate, const Expr *V,
3761  const Expr *X, const Expr *E,
3762  const Expr *UE, bool IsXLHSInRHSPart,
3763  SourceLocation Loc) {
3764  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3765  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3766  RValue NewVVal;
3767  LValue VLValue = CGF.EmitLValue(V);
3768  LValue XLValue = CGF.EmitLValue(X);
3769  RValue ExprRValue = CGF.EmitAnyExpr(E);
3770  llvm::AtomicOrdering AO = IsSeqCst
3771  ? llvm::AtomicOrdering::SequentiallyConsistent
3772  : llvm::AtomicOrdering::Monotonic;
3773  QualType NewVValType;
3774  if (UE) {
3775  // 'x' is updated with some additional value.
3776  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3777  "Update expr in 'atomic capture' must be a binary operator.");
3778  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3779  // Update expressions are allowed to have the following forms:
3780  // x binop= expr; -> xrval + expr;
3781  // x++, ++x -> xrval + 1;
3782  // x--, --x -> xrval - 1;
3783  // x = x binop expr; -> xrval binop expr
3784  // x = expr Op x; - > expr binop xrval;
3785  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3786  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3787  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3788  NewVValType = XRValExpr->getType();
3789  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3790  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3791  IsPostfixUpdate](RValue XRValue) {
3792  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3793  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3794  RValue Res = CGF.EmitAnyExpr(UE);
3795  NewVVal = IsPostfixUpdate ? XRValue : Res;
3796  return Res;
3797  };
3798  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3799  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3800  if (Res.first) {
3801  // 'atomicrmw' instruction was generated.
3802  if (IsPostfixUpdate) {
3803  // Use old value from 'atomicrmw'.
3804  NewVVal = Res.second;
3805  } else {
3806  // 'atomicrmw' does not provide new value, so evaluate it using old
3807  // value of 'x'.
3808  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3809  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3810  NewVVal = CGF.EmitAnyExpr(UE);
3811  }
3812  }
3813  } else {
3814  // 'x' is simply rewritten with some 'expr'.
3815  NewVValType = X->getType().getNonReferenceType();
3816  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3817  X->getType().getNonReferenceType(), Loc);
3818  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3819  NewVVal = XRValue;
3820  return ExprRValue;
3821  };
3822  // Try to perform atomicrmw xchg, otherwise simple exchange.
3823  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3824  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3825  Loc, Gen);
3826  if (Res.first) {
3827  // 'atomicrmw' instruction was generated.
3828  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3829  }
3830  }
3831  // Emit post-update store to 'v' of old/new 'x' value.
3832  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3833  // OpenMP, 2.12.6, atomic Construct
3834  // Any atomic construct with a seq_cst clause forces the atomically
3835  // performed operation to include an implicit flush operation without a
3836  // list.
3837  if (IsSeqCst)
3838  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3839 }
3840 
3842  bool IsSeqCst, bool IsPostfixUpdate,
3843  const Expr *X, const Expr *V, const Expr *E,
3844  const Expr *UE, bool IsXLHSInRHSPart,
3845  SourceLocation Loc) {
3846  switch (Kind) {
3847  case OMPC_read:
3848  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3849  break;
3850  case OMPC_write:
3851  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3852  break;
3853  case OMPC_unknown:
3854  case OMPC_update:
3855  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3856  break;
3857  case OMPC_capture:
3858  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3859  IsXLHSInRHSPart, Loc);
3860  break;
3861  case OMPC_if:
3862  case OMPC_final:
3863  case OMPC_num_threads:
3864  case OMPC_private:
3865  case OMPC_firstprivate:
3866  case OMPC_lastprivate:
3867  case OMPC_reduction:
3868  case OMPC_task_reduction:
3869  case OMPC_in_reduction:
3870  case OMPC_safelen:
3871  case OMPC_simdlen:
3872  case OMPC_collapse:
3873  case OMPC_default:
3874  case OMPC_seq_cst:
3875  case OMPC_shared:
3876  case OMPC_linear:
3877  case OMPC_aligned:
3878  case OMPC_copyin:
3879  case OMPC_copyprivate:
3880  case OMPC_flush:
3881  case OMPC_proc_bind:
3882  case OMPC_schedule:
3883  case OMPC_ordered:
3884  case OMPC_nowait:
3885  case OMPC_untied:
3886  case OMPC_threadprivate:
3887  case OMPC_depend:
3888  case OMPC_mergeable:
3889  case OMPC_device:
3890  case OMPC_threads:
3891  case OMPC_simd:
3892  case OMPC_map:
3893  case OMPC_num_teams:
3894  case OMPC_thread_limit:
3895  case OMPC_priority:
3896  case OMPC_grainsize:
3897  case OMPC_nogroup:
3898  case OMPC_num_tasks:
3899  case OMPC_hint:
3900  case OMPC_dist_schedule:
3901  case OMPC_defaultmap:
3902  case OMPC_uniform:
3903  case OMPC_to:
3904  case OMPC_from:
3905  case OMPC_use_device_ptr:
3906  case OMPC_is_device_ptr:
3907  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3908  }
3909 }
3910 
3912  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3914  for (const OMPClause *C : S.clauses()) {
3915  // Find first clause (skip seq_cst clause, if it is first).
3916  if (C->getClauseKind() != OMPC_seq_cst) {
3917  Kind = C->getClauseKind();
3918  break;
3919  }
3920  }
3921 
3922  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
3923  if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS))
3924  enterFullExpression(EWC);
3925  // Processing for statements under 'atomic capture'.
3926  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3927  for (const Stmt *C : Compound->body()) {
3928  if (const auto *EWC = dyn_cast<ExprWithCleanups>(C))
3929  enterFullExpression(EWC);
3930  }
3931  }
3932 
3933  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3934  PrePostActionTy &) {
3935  CGF.EmitStopPoint(CS);
3936  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3937  S.getV(), S.getExpr(), S.getUpdateExpr(),
3938  S.isXLHSInRHSPart(), S.getLocStart());
3939  };
3940  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3941  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
3942 }
3943 
3945  const OMPExecutableDirective &S,
3946  const RegionCodeGenTy &CodeGen) {
3948  CodeGenModule &CGM = CGF.CGM;
3949 
3950  // On device emit this construct as inlined code.
3951  if (CGM.getLangOpts().OpenMPIsDevice) {
3952  OMPLexicalScope Scope(CGF, S, OMPD_target);
3953  CGM.getOpenMPRuntime().emitInlinedDirective(
3954  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3956  });
3957  return;
3958  }
3959 
3960  llvm::Function *Fn = nullptr;
3961  llvm::Constant *FnID = nullptr;
3962 
3963  const Expr *IfCond = nullptr;
3964  // Check for the at most one if clause associated with the target region.
3965  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3966  if (C->getNameModifier() == OMPD_unknown ||
3967  C->getNameModifier() == OMPD_target) {
3968  IfCond = C->getCondition();
3969  break;
3970  }
3971  }
3972 
3973  // Check if we have any device clause associated with the directive.
3974  const Expr *Device = nullptr;
3975  if (auto *C = S.getSingleClause<OMPDeviceClause>())
3976  Device = C->getDevice();
3977 
3978  // Check if we have an if clause whose conditional always evaluates to false
3979  // or if we do not have any targets specified. If so the target region is not
3980  // an offload entry point.
3981  bool IsOffloadEntry = true;
3982  if (IfCond) {
3983  bool Val;
3984  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
3985  IsOffloadEntry = false;
3986  }
3987  if (CGM.getLangOpts().OMPTargetTriples.empty())
3988  IsOffloadEntry = false;
3989 
3990  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
3991  StringRef ParentName;
3992  // In case we have Ctors/Dtors we use the complete type variant to produce
3993  // the mangling of the device outlined kernel.
3994  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
3995  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
3996  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
3997  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3998  else
3999  ParentName =
4000  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4001 
4002  // Emit target region as a standalone region.
4003  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4004  IsOffloadEntry, CodeGen);
4005  OMPLexicalScope Scope(CGF, S, OMPD_task);
4006  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4007 }
4008 
4010  PrePostActionTy &Action) {
4011  Action.Enter(CGF);
4012  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4013  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4014  CGF.EmitOMPPrivateClause(S, PrivateScope);
4015  (void)PrivateScope.Privatize();
4016 
4017  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4018 }
4019 
4021  StringRef ParentName,
4022  const OMPTargetDirective &S) {
4023  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4024  emitTargetRegion(CGF, S, Action);
4025  };
4026  llvm::Function *Fn;
4027  llvm::Constant *Addr;
4028  // Emit target region as a standalone region.
4029  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4030  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4031  assert(Fn && Addr && "Target device function emission failed.");
4032 }
4033 
4035  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4036  emitTargetRegion(CGF, S, Action);
4037  };
4038  emitCommonOMPTargetDirective(*this, S, CodeGen);
4039 }
4040 
4042  const OMPExecutableDirective &S,
4043  OpenMPDirectiveKind InnermostKind,
4044  const RegionCodeGenTy &CodeGen) {
4045  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4046  llvm::Value *OutlinedFn =
4047  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4048  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4049 
4050  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4051  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4052  if (NT || TL) {
4053  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4054  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4055 
4056  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4057  S.getLocStart());
4058  }
4059 
4060  OMPTeamsScope Scope(CGF, S);
4062  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4063  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
4064  CapturedVars);
4065 }
4066 
4068  // Emit teams region as a standalone region.
4069  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4070  Action.Enter(CGF);
4071  OMPPrivateScope PrivateScope(CGF);
4072  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4073  CGF.EmitOMPPrivateClause(S, PrivateScope);
4074  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4075  (void)PrivateScope.Privatize();
4076  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4077  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4078  };
4079  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4081  [](CodeGenFunction &) { return nullptr; });
4082 }
4083 
4085  const OMPTargetTeamsDirective &S) {
4086  auto *CS = S.getCapturedStmt(OMPD_teams);
4087  Action.Enter(CGF);
4088  // Emit teams region as a standalone region.
4089  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4090  Action.Enter(CGF);
4091  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4092  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4093  CGF.EmitOMPPrivateClause(S, PrivateScope);
4094  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4095  (void)PrivateScope.Privatize();
4096  CGF.EmitStmt(CS->getCapturedStmt());
4097  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4098  };
4099  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4101  [](CodeGenFunction &) { return nullptr; });
4102 }
4103 
4105  CodeGenModule &CGM, StringRef ParentName,
4106  const OMPTargetTeamsDirective &S) {
4107  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4108  emitTargetTeamsRegion(CGF, Action, S);
4109  };
4110  llvm::Function *Fn;
4111  llvm::Constant *Addr;
4112  // Emit target region as a standalone region.
4113  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4114  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4115  assert(Fn && Addr && "Target device function emission failed.");
4116 }
4117 
4119  const OMPTargetTeamsDirective &S) {
4120  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4121  emitTargetTeamsRegion(CGF, Action, S);
4122  };
4123  emitCommonOMPTargetDirective(*this, S, CodeGen);
4124 }
4125 
4126 static void
4129  Action.Enter(CGF);
4130  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4132  };
4133 
4134  // Emit teams region as a standalone region.
4135  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4136  PrePostActionTy &Action) {
4137  Action.Enter(CGF);
4138  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4139  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4140  (void)PrivateScope.Privatize();
4141  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4142  CodeGenDistribute);
4143  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4144  };
4145  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4147  [](CodeGenFunction &) { return nullptr; });
4148 }
4149 
4151  CodeGenModule &CGM, StringRef ParentName,
4153  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4154  emitTargetTeamsDistributeRegion(CGF, Action, S);
4155  };
4156  llvm::Function *Fn;
4157  llvm::Constant *Addr;
4158  // Emit target region as a standalone region.
4159  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4160  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4161  assert(Fn && Addr && "Target device function emission failed.");
4162 }
4163 
4166  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4167  emitTargetTeamsDistributeRegion(CGF, Action, S);
4168  };
4169  emitCommonOMPTargetDirective(*this, S, CodeGen);
4170 }
4171 
4173  CodeGenFunction &CGF, PrePostActionTy &Action,
4175  Action.Enter(CGF);
4176  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4178  };
4179 
4180  // Emit teams region as a standalone region.
4181  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4182  PrePostActionTy &Action) {
4183  Action.Enter(CGF);
4184  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4185  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4186  (void)PrivateScope.Privatize();
4187  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4188  CodeGenDistribute);
4189  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4190  };
4191  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4193  [](CodeGenFunction &) { return nullptr; });
4194 }
4195 
4197  CodeGenModule &CGM, StringRef ParentName,
4199  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4200  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4201  };
4202  llvm::Function *Fn;
4203  llvm::Constant *Addr;
4204  // Emit target region as a standalone region.
4205  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4206  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4207  assert(Fn && Addr && "Target device function emission failed.");
4208 }
4209 
4212  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4213  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4214  };
4215  emitCommonOMPTargetDirective(*this, S, CodeGen);
4216 }
4217 
4219  const OMPTeamsDistributeDirective &S) {
4220 
4221  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4223  };
4224 
4225  // Emit teams region as a standalone region.
4226  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4227  PrePostActionTy &Action) {
4228  Action.Enter(CGF);
4229  OMPPrivateScope PrivateScope(CGF);
4230  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4231  (void)PrivateScope.Privatize();
4232  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4233  CodeGenDistribute);
4234  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4235  };
4236  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4238  [](CodeGenFunction &) { return nullptr; });
4239 }
4240 
4243  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4245  };
4246 
4247  // Emit teams region as a standalone region.
4248  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4249  PrePostActionTy &Action) {
4250  Action.Enter(CGF);
4251  OMPPrivateScope PrivateScope(CGF);
4252  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4253  (void)PrivateScope.Privatize();
4254  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4255  CodeGenDistribute);
4256  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4257  };
4258  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4260  [](CodeGenFunction &) { return nullptr; });
4261 }
4262 
4265  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4267  S.getDistInc());
4268  };
4269 
4270  // Emit teams region as a standalone region.
4271  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4272  PrePostActionTy &Action) {
4273  Action.Enter(CGF);
4274  OMPPrivateScope PrivateScope(CGF);
4275  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4276  (void)PrivateScope.Privatize();
4277  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4278  CodeGenDistribute);
4279  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4280  };
4281  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4283  [](CodeGenFunction &) { return nullptr; });
4284 }
4285 
4288  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4290  S.getDistInc());
4291  };
4292 
4293  // Emit teams region as a standalone region.
4294  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4295  PrePostActionTy &Action) {
4296  Action.Enter(CGF);
4297  OMPPrivateScope PrivateScope(CGF);
4298  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4299  (void)PrivateScope.Privatize();
4300  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4301  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4302  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4303  };
4304  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4306  [](CodeGenFunction &) { return nullptr; });
4307 }
4308 
4311  PrePostActionTy &Action) {
4312  Action.Enter(CGF);
4313  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4315  S.getDistInc());
4316  };
4317 
4318  // Emit teams region as a standalone region.
4319  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4320  PrePostActionTy &Action) {
4321  Action.Enter(CGF);
4322  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4323  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4324  (void)PrivateScope.Privatize();
4325  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4326  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4327  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4328  };
4329 
4330  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4331  CodeGenTeams);
4333  [](CodeGenFunction &) { return nullptr; });
4334 }
4335 
4337  CodeGenModule &CGM, StringRef ParentName,
4339  // Emit SPMD target teams distribute parallel for region as a standalone
4340  // region.
4341  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4343  };
4344  llvm::Function *Fn;
4345  llvm::Constant *Addr;
4346  // Emit target region as a standalone region.
4347  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4348  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4349  assert(Fn && Addr && "Target device function emission failed.");
4350 }
4351 
4354  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4356  };
4357  emitCommonOMPTargetDirective(*this, S, CodeGen);
4358 }
4359 
4361  CodeGenFunction &CGF,
4363  PrePostActionTy &Action) {
4364  Action.Enter(CGF);
4365  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4367  S.getDistInc());
4368  };
4369 
4370  // Emit teams region as a standalone region.
4371  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4372  PrePostActionTy &Action) {
4373  Action.Enter(CGF);
4374  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4375  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4376  (void)PrivateScope.Privatize();
4377  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4378  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4379  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4380  };
4381 
4382  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4383  CodeGenTeams);
4385  [](CodeGenFunction &) { return nullptr; });
4386 }
4387 
4389  CodeGenModule &CGM, StringRef ParentName,
4391  // Emit SPMD target teams distribute parallel for simd region as a standalone
4392  // region.
4393  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4395  };
4396  llvm::Function *Fn;
4397  llvm::Constant *Addr;
4398  // Emit target region as a standalone region.
4399  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4400  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4401  assert(Fn && Addr && "Target device function emission failed.");
4402 }
4403 
4406  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4408  };
4409  emitCommonOMPTargetDirective(*this, S, CodeGen);
4410 }
4411 
4413  const OMPCancellationPointDirective &S) {
4414  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
4415  S.getCancelRegion());
4416 }
4417 
4419  const Expr *IfCond = nullptr;
4420  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4421  if (C->getNameModifier() == OMPD_unknown ||
4422  C->getNameModifier() == OMPD_cancel) {
4423  IfCond = C->getCondition();
4424  break;
4425  }
4426  }
4427  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
4428  S.getCancelRegion());
4429 }
4430 
4433  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4434  Kind == OMPD_target_parallel)
4435  return ReturnBlock;
4436  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4437  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4438  Kind == OMPD_distribute_parallel_for ||
4439  Kind == OMPD_target_parallel_for ||
4440  Kind == OMPD_teams_distribute_parallel_for ||
4441  Kind == OMPD_target_teams_distribute_parallel_for);
4442  return OMPCancelStack.getExitBlock();
4443 }
4444 
4446  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4447  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4448  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4449  auto OrigVarIt = C.varlist_begin();
4450  auto InitIt = C.inits().begin();
4451  for (const Expr *PvtVarIt : C.private_copies()) {
4452  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4453  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4454  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4455 
4456  // In order to identify the right initializer we need to match the
4457  // declaration used by the mapping logic. In some cases we may get
4458  // OMPCapturedExprDecl that refers to the original declaration.
4459  const ValueDecl *MatchingVD = OrigVD;
4460  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4461  // OMPCapturedExprDecl are used to privative fields of the current
4462  // structure.
4463  const auto *ME = cast<MemberExpr>(OED->getInit());
4464  assert(isa<CXXThisExpr>(ME->getBase()) &&
4465  "Base should be the current struct!");
4466  MatchingVD = ME->getMemberDecl();
4467  }
4468 
4469  // If we don't have information about the current list item, move on to
4470  // the next one.
4471  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4472  if (InitAddrIt == CaptureDeviceAddrMap.end())
4473  continue;
4474 
4475  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4476  InitAddrIt, InitVD,
4477  PvtVD]() {
4478  // Initialize the temporary initialization variable with the address we
4479  // get from the runtime library. We have to cast the source address
4480  // because it is always a void *. References are materialized in the
4481  // privatization scope, so the initialization here disregards the fact
4482  // the original variable is a reference.
4483  QualType AddrQTy =
4484  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4485  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4486  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4487  setAddrOfLocalVar(InitVD, InitAddr);
4488 
4489  // Emit private declaration, it will be initialized by the value we
4490  // declaration we just added to the local declarations map.
4491  EmitDecl(*PvtVD);
4492 
4493  // The initialization variables reached its purpose in the emission
4494  // of the previous declaration, so we don't need it anymore.
4495  LocalDeclMap.erase(InitVD);
4496 
4497  // Return the address of the private variable.
4498  return GetAddrOfLocalVar(PvtVD);
4499  });
4500  assert(IsRegistered && "firstprivate var already registered as private");
4501  // Silence the warning about unused variable.
4502  (void)IsRegistered;
4503 
4504  ++OrigVarIt;
4505  ++InitIt;
4506  }
4507 }
4508 
4509 // Generate the instructions for '#pragma omp target data' directive.
4511  const OMPTargetDataDirective &S) {
4512  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4513 
4514  // Create a pre/post action to signal the privatization of the device pointer.
4515  // This action can be replaced by the OpenMP runtime code generation to
4516  // deactivate privatization.
4517  bool PrivatizeDevicePointers = false;
4518  class DevicePointerPrivActionTy : public PrePostActionTy {
4519  bool &PrivatizeDevicePointers;
4520 
4521  public:
4522  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4523  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4524  void Enter(CodeGenFunction &CGF) override {
4525  PrivatizeDevicePointers = true;
4526  }
4527  };
4528  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4529 
4530  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4531  CodeGenFunction &CGF, PrePostActionTy &Action) {
4532  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4534  };
4535 
4536  // Codegen that selects whether to generate the privatization code or not.
4537  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4538  &InnermostCodeGen](CodeGenFunction &CGF,
4539  PrePostActionTy &Action) {
4540  RegionCodeGenTy RCG(InnermostCodeGen);
4541  PrivatizeDevicePointers = false;
4542 
4543  // Call the pre-action to change the status of PrivatizeDevicePointers if
4544  // needed.
4545  Action.Enter(CGF);
4546 
4547  if (PrivatizeDevicePointers) {
4548  OMPPrivateScope PrivateScope(CGF);
4549  // Emit all instances of the use_device_ptr clause.
4550  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4551  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4552  Info.CaptureDeviceAddrMap);
4553  (void)PrivateScope.Privatize();
4554  RCG(CGF);
4555  } else {
4556  RCG(CGF);
4557  }
4558  };
4559 
4560  // Forward the provided action to the privatization codegen.
4561  RegionCodeGenTy PrivRCG(PrivCodeGen);
4562  PrivRCG.setAction(Action);
4563 
4564  // Notwithstanding the body of the region is emitted as inlined directive,
4565  // we don't use an inline scope as changes in the references inside the
4566  // region are expected to be visible outside, so we do not privative them.
4567  OMPLexicalScope Scope(CGF, S);
4568  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4569  PrivRCG);
4570  };
4571 
4572  RegionCodeGenTy RCG(CodeGen);
4573 
4574  // If we don't have target devices, don't bother emitting the data mapping
4575  // code.
4576  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4577  RCG(*this);
4578  return;
4579  }
4580 
4581  // Check if we have any if clause associated with the directive.
4582  const Expr *IfCond = nullptr;
4583  if (const auto *C = S.getSingleClause<OMPIfClause>())
4584  IfCond = C->getCondition();
4585 
4586  // Check if we have any device clause associated with the directive.
4587  const Expr *Device = nullptr;
4588  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4589  Device = C->getDevice();
4590 
4591  // Set the action to signal privatization of device pointers.
4592  RCG.setAction(PrivAction);
4593 
4594  // Emit region code.
4595  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4596  Info);
4597 }
4598 
4600  const OMPTargetEnterDataDirective &S) {
4601  // If we don't have target devices, don't bother emitting the data mapping
4602  // code.
4603  if (CGM.getLangOpts().OMPTargetTriples.empty())
4604  return;
4605 
4606  // Check if we have any if clause associated with the directive.
4607  const Expr *IfCond = nullptr;
4608  if (const auto *C = S.getSingleClause<OMPIfClause>())
4609  IfCond = C->getCondition();
4610 
4611  // Check if we have any device clause associated with the directive.
4612  const Expr *Device = nullptr;
4613  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4614  Device = C->getDevice();
4615 
4616  OMPLexicalScope Scope(*this, S, OMPD_task);
4617  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4618 }
4619 
4621  const OMPTargetExitDataDirective &S) {
4622  // If we don't have target devices, don't bother emitting the data mapping
4623  // code.
4624  if (CGM.getLangOpts().OMPTargetTriples.empty())
4625  return;
4626 
4627  // Check if we have any if clause associated with the directive.
4628  const Expr *IfCond = nullptr;
4629  if (const auto *C = S.getSingleClause<OMPIfClause>())
4630  IfCond = C->getCondition();
4631 
4632  // Check if we have any device clause associated with the directive.
4633  const Expr *Device = nullptr;
4634  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4635  Device = C->getDevice();
4636 
4637  OMPLexicalScope Scope(*this, S, OMPD_task);
4638  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4639 }
4640 
4642  const OMPTargetParallelDirective &S,
4643  PrePostActionTy &Action) {
4644  // Get the captured statement associated with the 'parallel' region.
4645  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4646  Action.Enter(CGF);
4647  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4648  Action.Enter(CGF);
4649  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4650  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4651  CGF.EmitOMPPrivateClause(S, PrivateScope);
4652  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4653  (void)PrivateScope.Privatize();
4654  // TODO: Add support for clauses.
4655  CGF.EmitStmt(CS->getCapturedStmt());
4656  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4657  };
4658  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4661  [](CodeGenFunction &) { return nullptr; });
4662 }
4663 
4665  CodeGenModule &CGM, StringRef ParentName,
4666  const OMPTargetParallelDirective &S) {
4667  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4668  emitTargetParallelRegion(CGF, S, Action);
4669  };
4670  llvm::Function *Fn;
4671  llvm::Constant *Addr;
4672  // Emit target region as a standalone region.
4673  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4674  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4675  assert(Fn && Addr && "Target device function emission failed.");
4676 }
4677 
4679  const OMPTargetParallelDirective &S) {
4680  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4681  emitTargetParallelRegion(CGF, S, Action);
4682  };
4683  emitCommonOMPTargetDirective(*this, S, CodeGen);
4684 }
4685 
4688  PrePostActionTy &Action) {
4689  Action.Enter(CGF);
4690  // Emit directive as a combined directive that consists of two implicit
4691  // directives: 'parallel' with 'for' directive.
4692  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4693  Action.Enter(CGF);
4695  CGF, OMPD_target_parallel_for, S.hasCancel());
4698  };
4699  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4701 }
4702 
4704  CodeGenModule &CGM, StringRef ParentName,
4705  const OMPTargetParallelForDirective &S) {
4706  // Emit SPMD target parallel for region as a standalone region.
4707  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4708  emitTargetParallelForRegion(CGF, S, Action);
4709  };
4710  llvm::Function *Fn;
4711  llvm::Constant *Addr;
4712  // Emit target region as a standalone region.
4713  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4714  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4715  assert(Fn && Addr && "Target device function emission failed.");
4716 }
4717 
4719  const OMPTargetParallelForDirective &S) {
4720  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4721  emitTargetParallelForRegion(CGF, S, Action);
4722  };
4723  emitCommonOMPTargetDirective(*this, S, CodeGen);
4724 }
4725 
4726 static void
4729  PrePostActionTy &Action) {
4730  Action.Enter(CGF);
4731  // Emit directive as a combined directive that consists of two implicit
4732  // directives: 'parallel' with 'for' directive.
4733  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4734  Action.Enter(CGF);
4737  };
4738  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4740 }
4741 
4743  CodeGenModule &CGM, StringRef ParentName,
4745  // Emit SPMD target parallel for region as a standalone region.
4746  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4747  emitTargetParallelForSimdRegion(CGF, S, Action);
4748  };
4749  llvm::Function *Fn;
4750  llvm::Constant *Addr;
4751  // Emit target region as a standalone region.
4752  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4753  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4754  assert(Fn && Addr && "Target device function emission failed.");
4755 }
4756 
4759  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4760  emitTargetParallelForSimdRegion(CGF, S, Action);
4761  };
4762  emitCommonOMPTargetDirective(*this, S, CodeGen);
4763 }
4764 
4765 /// Emit a helper variable and return corresponding lvalue.
4766 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4767  const ImplicitParamDecl *PVD,
4769  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4770  Privates.addPrivate(VDecl,
4771  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4772 }
4773 
4776  // Emit outlined function for task construct.
4777  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4778  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4779  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4780  const Expr *IfCond = nullptr;
4781  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4782  if (C->getNameModifier() == OMPD_unknown ||
4783  C->getNameModifier() == OMPD_taskloop) {
4784  IfCond = C->getCondition();
4785  break;
4786  }
4787  }
4788 
4789  OMPTaskDataTy Data;
4790  // Check if taskloop must be emitted without taskgroup.
4792  // TODO: Check if we should emit tied or untied task.
4793  Data.Tied = true;
4794  // Set scheduling for taskloop
4795  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4796  // grainsize clause
4797  Data.Schedule.setInt(/*IntVal=*/false);
4798  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4799  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4800  // num_tasks clause
4801  Data.Schedule.setInt(/*IntVal=*/true);
4802  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4803  }
4804 
4805  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4806  // if (PreCond) {
4807  // for (IV in 0..LastIteration) BODY;
4808  // <Final counter/linear vars updates>;
4809  // }
4810  //
4811 
4812  // Emit: if (PreCond) - begin.
4813  // If the condition constant folds and can be elided, avoid emitting the
4814  // whole loop.
4815  bool CondConstant;
4816  llvm::BasicBlock *ContBlock = nullptr;
4817  OMPLoopScope PreInitScope(CGF, S);
4818  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4819  if (!CondConstant)
4820  return;
4821  } else {
4822  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4823  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4824  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4825  CGF.getProfileCount(&S));
4826  CGF.EmitBlock(ThenBlock);
4827  CGF.incrementProfileCounter(&S);
4828  }
4829 
4831  CGF.EmitOMPSimdInit(S);
4832 
4833  OMPPrivateScope LoopScope(CGF);
4834  // Emit helper vars inits.
4835  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4836  auto *I = CS->getCapturedDecl()->param_begin();
4837  auto *LBP = std::next(I, LowerBound);
4838  auto *UBP = std::next(I, UpperBound);
4839  auto *STP = std::next(I, Stride);
4840  auto *LIP = std::next(I, LastIter);
4841  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4842  LoopScope);
4843  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4844  LoopScope);
4845  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4846  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4847  LoopScope);
4848  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4849  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4850  (void)LoopScope.Privatize();
4851  // Emit the loop iteration variable.
4852  const Expr *IVExpr = S.getIterationVariable();
4853  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4854  CGF.EmitVarDecl(*IVDecl);
4855  CGF.EmitIgnoredExpr(S.getInit());
4856 
4857  // Emit the iterations count variable.
4858  // If it is not a variable, Sema decided to calculate iterations count on
4859  // each iteration (e.g., it is foldable into a constant).
4860  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4861  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4862  // Emit calculation of the iterations count.
4863  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4864  }
4865 
4866  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4867  S.getInc(),
4868  [&S](CodeGenFunction &CGF) {
4869  CGF.EmitOMPLoopBody(S, JumpDest());
4870  CGF.EmitStopPoint(&S);
4871  },
4872  [](CodeGenFunction &) {});
4873  // Emit: if (PreCond) - end.
4874  if (ContBlock) {
4875  CGF.EmitBranch(ContBlock);
4876  CGF.EmitBlock(ContBlock, true);
4877  }
4878  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4879  if (HasLastprivateClause) {
4880  CGF.EmitOMPLastprivateClauseFinal(
4882  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4883  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4884  (*LIP)->getType(), S.getLocStart())));
4885  }
4886  };
4887  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4888  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4889  const OMPTaskDataTy &Data) {
4890  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4891  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4892  OMPLoopScope PreInitScope(CGF, S);
4893  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
4894  OutlinedFn, SharedsTy,
4895  CapturedStruct, IfCond, Data);
4896  };
4897  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4898  CodeGen);
4899  };
4900  if (Data.Nogroup) {
4901  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4902  } else {
4903  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4904  *this,
4905  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4906  PrePostActionTy &Action) {
4907  Action.Enter(CGF);
4908  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
4909  Data);
4910  },
4911  S.getLocStart());
4912  }
4913 }
4914 
4916  EmitOMPTaskLoopBasedDirective(S);
4917 }
4918 
4920  const OMPTaskLoopSimdDirective &S) {
4921  EmitOMPTaskLoopBasedDirective(S);
4922 }
4923 
4924 // Generate the instructions for '#pragma omp target update' directive.
4926  const OMPTargetUpdateDirective &S) {
4927  // If we don't have target devices, don't bother emitting the data mapping
4928  // code.
4929  if (CGM.getLangOpts().OMPTargetTriples.empty())
4930  return;
4931 
4932  // Check if we have any if clause associated with the directive.
4933  const Expr *IfCond = nullptr;
4934  if (const auto *C = S.getSingleClause<OMPIfClause>())
4935  IfCond = C->getCondition();
4936 
4937  // Check if we have any device clause associated with the directive.
4938  const Expr *Device = nullptr;
4939  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4940  Device = C->getDevice();
4941 
4942  OMPLexicalScope Scope(*this, S, OMPD_task);
4943  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4944 }
4945 
4947  const OMPExecutableDirective &D) {
4948  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
4949  return;
4950  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
4952  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
4953  } else {
4954  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
4955  for (const Expr *E : LD->counters()) {
4956  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4957  cast<DeclRefExpr>(E)->getDecl())) {
4958  // Emit only those that were not explicitly referenced in clauses.
4959  if (!CGF.LocalDeclMap.count(VD))
4960  CGF.EmitVarDecl(*VD);
4961  }
4962  }
4963  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
4964  if (!C->getNumForLoops())
4965  continue;
4966  for (unsigned I = LD->getCollapsedNumber(),
4967  E = C->getLoopNumIterations().size();
4968  I < E; ++I) {
4969  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4970  cast<DeclRefExpr>(C->getLoopCunter(I))->getDecl())) {
4971  // Emit only those that were not explicitly referenced in clauses.
4972  if (!CGF.LocalDeclMap.count(VD))
4973  CGF.EmitVarDecl(*VD);
4974  }
4975  }
4976  }
4977  }
4979  }
4980  };
4981  OMPSimdLexicalScope Scope(*this, D);
4982  CGM.getOpenMPRuntime().emitInlinedDirective(
4983  *this,
4984  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
4985  : D.getDirectiveKind(),
4986  CodeGen);
4987 }
4988 
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:653
bool isAggregate() const
Definition: CGValue.h:54
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3216
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:845
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1399
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1739
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1716
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:813
Other implicit parameter.
Definition: Decl.h:1495
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:26
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2393
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:124
A (possibly-)qualified type.
Definition: Type.h:655
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:2299
bool isArrayType() const
Definition: Type.h:6162
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:262
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:156
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4117
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:805
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2100
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:497
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1149
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3627
bool isNothrow() const
Definition: Decl.cpp:4355
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:242
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:228
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6062
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:301
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1428
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3764
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1865
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:2243
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2668
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:679
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2277
Floating point control options.
Definition: LangOptions.h:263
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1520
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3832
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:897
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1885
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:687
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2431
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
Definition: Decl.h:814
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2614
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:384
const Stmt * getBody() const
Definition: StmtOpenMP.h:927
Extra information about a function prototype.
Definition: Type.h:3551
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:891
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:773
llvm::Value * getPointer() const
Definition: Address.h:38
unsigned getContextParamPosition() const
Definition: Decl.h:4111
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2228
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:449
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition: Decl.h:269
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:3016
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
Represents a struct/union/class.
Definition: Decl.h:3570
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents &#39;#pragma omp parallel&#39; directive.
Definition: StmtOpenMP.h:278
Address getAddress() const
Definition: CGValue.h:327
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:150
A C++ nested-name-specifier augmented with source location information.
This represents &#39;simd&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
Definition: Decl.h:3786
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Expr * getEnsureUpperBound() const
Definition: StmtOpenMP.h:829
SmallVector< const Expr *, 4 > LastprivateVars
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition: CGExpr.cpp:2255
Represents a member of a struct/union/class.
Definition: Decl.h:2534
This represents clause &#39;lastprivate&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
CharUnits getAlignment() const
Definition: CGValue.h:316
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
Definition: StmtOpenMP.h:211
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Expr * getInc() const
Definition: StmtOpenMP.h:789
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:957
bool isReferenceType() const
Definition: Type.h:6125
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr)
Definition: Expr.cpp:391
This represents &#39;#pragma omp target simd&#39; directive.
Definition: StmtOpenMP.h:3352
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:246
This represents &#39;#pragma omp barrier&#39; directive.
Definition: StmtOpenMP.h:1851
This is a common base class for loop directives (&#39;omp simd&#39;, &#39;omp for&#39;, &#39;omp for simd&#39; etc...
Definition: StmtOpenMP.h:340
This represents &#39;#pragma omp critical&#39; directive.
Definition: StmtOpenMP.h:1446
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2730
IdentifierTable & Idents
Definition: ASTContext.h:545
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:110
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition: CGExpr.cpp:194
This represents clause &#39;copyprivate&#39; in the &#39;#pragma omp ...&#39; directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for &#39;dist_schedule&#39; clause.
Definition: OpenMPKinds.h:100
This represents &#39;#pragma omp distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3067
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:4352
This represents &#39;#pragma omp teams distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3556
BinaryOperatorKind
static bool hasScalarEvaluationKind(QualType T)
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:2286
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:963
Expr * getIsLastIterVariable() const
Definition: StmtOpenMP.h:797
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
OpenMPScheduleClauseKind Schedule
Definition: OpenMPKinds.h:125
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
Definition: TargetInfo.h:602
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Expr * getX()
Get &#39;x&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2212
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
Definition: Type.h:6575
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
SourceLocation getLocEnd() const LLVM_READONLY
Definition: Stmt.h:2322
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:4119
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:67
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:765
bool isComplex() const
Definition: CGValue.h:53
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:3143
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:5959
OpenMPScheduleClauseModifier M2
Definition: OpenMPKinds.h:127
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:147
SmallVector< const Expr *, 4 > PrivateCopies
SourceLocation getLocStart() const LLVM_READONLY
Returns starting location of directive kind.
Definition: StmtOpenMP.h:168
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents &#39;#pragma omp cancellation point&#39; directive.
Definition: StmtOpenMP.h:2686
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Expr * getSizeExpr() const
Definition: Type.h:2847
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:40
field_iterator field_begin() const
Definition: Decl.cpp:4040
This represents &#39;final&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:332
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents &#39;#pragma omp teams&#39; directive.
Definition: StmtOpenMP.h:2629
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:182
SmallVector< const Expr *, 4 > FirstprivateCopies
Expr * getInit() const
Definition: StmtOpenMP.h:785
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This represents clause &#39;reduction&#39; in the &#39;#pragma omp ...&#39; directives.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
This represents &#39;#pragma omp teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3486
void ForceCleanup(std::initializer_list< llvm::Value **> ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool isSimple() const
Definition: CGValue.h:252
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
An ordinary object is located at an address in memory.
Definition: Specifiers.h:126
SmallVector< const Expr *, 4 > ReductionOps
Controls insertion of cancellation exit blocks in worksharing constructs.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler&#39;s counter for the given statement by StepV.
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
Represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:4037
SmallVector< const Expr *, 4 > ReductionVars
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
Definition: Decl.h:1911
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition: Decl.cpp:4290
This represents &#39;#pragma omp target parallel for simd&#39; directive.
Definition: StmtOpenMP.h:3284
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:945
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:616
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition: CGDecl.cpp:1152
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1329
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents &#39;#pragma omp taskgroup&#39; directive.
Definition: StmtOpenMP.h:1939
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:39
This represents clause &#39;aligned&#39; in the &#39;#pragma omp ...&#39; directives.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
Definition: Expr.cpp:1785
Class intended to support codegen of all kind of the reduction clauses.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Expr * getCombinedLowerBoundVariable() const
Definition: StmtOpenMP.h:885
The class used to assign some variables some temporarily addresses.
This represents clause &#39;task_reduction&#39; in the &#39;#pragma omp taskgroup&#39; directives.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
This represents &#39;#pragma omp distribute&#39; directive.
Definition: StmtOpenMP.h:2940
This represents implicit clause &#39;depend&#39; for the &#39;#pragma omp task&#39; directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:198
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1583
This represents &#39;proc_bind&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:677
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition: Decl.h:637
Expr - This represents one expression.
Definition: Expr.h:106
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Emit only debug info necessary for generating line number tables (-gline-tables-only).
void EmitAutoVarInit(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1432
static Address invalid()
Definition: Address.h:35
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
This represents &#39;simdlen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:503
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
Definition: Stmt.cpp:133
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:66
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:6589
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1133
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
This represents &#39;#pragma omp target teams distribute parallel for simd&#39; combined directive.
Definition: StmtOpenMP.h:3916
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:33
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Definition: StmtOpenMP.h:130
This represents &#39;#pragma omp target teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3989
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
This represents &#39;ordered&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:927
child_range children()
Definition: Stmt.cpp:1095
Expr * getDistInc() const
Definition: StmtOpenMP.h:873
Expr * getNextLowerBound() const
Definition: StmtOpenMP.h:837
QualType getType() const
Definition: Expr.h:128
Expr * getPrevEnsureUpperBound() const
Definition: StmtOpenMP.h:879
This represents &#39;#pragma omp for&#39; directive.
Definition: StmtOpenMP.h:1072
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
SmallVector< const Expr *, 4 > FirstprivateVars
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition: Decl.cpp:4340
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the &#39;for&#39; loop has a dispatch schedule (e.g.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:197
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:127
This represents &#39;#pragma omp target teams&#39; directive.
Definition: StmtOpenMP.h:3705
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
SourceLocation getEnd() const
UnaryOperator - This represents the unary-expression&#39;s (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
Definition: Expr.h:1805
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
Definition: ASTContext.h:1365
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
This represents &#39;#pragma omp cancel&#39; directive.
Definition: StmtOpenMP.h:2744
This represents clause &#39;firstprivate&#39; in the &#39;#pragma omp ...&#39; directives.
ValueDecl * getDecl()
Definition: Expr.h:1059
const LangOptions & getLangOpts() const
ASTContext & getContext() const
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1461
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition: Decl.cpp:2006
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:35
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
This represents &#39;#pragma omp flush&#39; directive.
Definition: StmtOpenMP.h:2012
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
Definition: StmtOpenMP.h:162
This represents &#39;#pragma omp parallel for simd&#39; directive.
Definition: StmtOpenMP.h:1600
This represents &#39;seq_cst&#39; clause in the &#39;#pragma omp atomic&#39; directive.
This represents &#39;untied&#39; clause in the &#39;#pragma omp ...&#39; directive.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition: DeclBase.h:992
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
void EmitAlignmentAssumption(llvm::Value *PtrValue, unsigned Alignment, llvm::Value *OffsetValue=nullptr)
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Expr * getLastIteration() const
Definition: StmtOpenMP.h:769
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
Definition: StmtOpenMP.h:148
bool isPostfixUpdate() const
Return true if &#39;v&#39; expression must be updated to original value of &#39;x&#39;, false if &#39;v&#39; must be updated ...
Definition: StmtOpenMP.h:2231
This represents &#39;#pragma omp target enter data&#39; directive.
Definition: StmtOpenMP.h:2372
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Expr * getStrideVariable() const
Definition: StmtOpenMP.h:821
This represents &#39;num_teams&#39; clause in the &#39;#pragma omp ...&#39; directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Definition: Expr.h:875
const Stmt * getPreInits() const
Definition: StmtOpenMP.h:793
Kind
This captures a statement into a function.
Definition: Stmt.h:2125
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
This represents &#39;#pragma omp single&#39; directive.
Definition: StmtOpenMP.h:1344
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents &#39;hint&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPForDirective(const OMPForDirective &S)
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Pseudo declaration for capturing expressions.
Definition: DeclOpenMP.h:187
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition: Type.h:1958
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:33
This represents &#39;schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:746
Expr * getExpr()
Get &#39;expr&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2238
SourceLocation getLocStart() const LLVM_READONLY
Definition: Stmt.h:401
OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:23
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
Definition: StmtOpenMP.h:1504
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
Definition: CGStmt.cpp:48
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition: Decl.h:1077
C-style initialization with assignment.
Definition: Decl.h:819
This represents &#39;#pragma omp taskwait&#39; directive.
Definition: StmtOpenMP.h:1895
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
This file defines OpenMP nodes for declarative directives.
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
This is a basic class for representing single OpenMP clause.
Definition: OpenMPClause.h:51
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
CanQualType VoidTy
Definition: ASTContext.h:1004
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
bool isAnyPointerType() const
Definition: Type.h:6117
virtual void Enter(CodeGenFunction &CGF)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
An aligned address.
Definition: Address.h:25
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition: Stmt.h:2226
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
Definition: Expr.h:249
This represents &#39;#pragma omp target&#39; directive.
Definition: StmtOpenMP.h:2256
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Expr * getV()
Get &#39;v&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2233
Complete object dtor.
Definition: ABI.h:36
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents &#39;#pragma omp ordered&#39; directive.
Definition: StmtOpenMP.h:2067
QualType getType() const
Definition: CGValue.h:264
This represents &#39;#pragma omp target update&#39; directive.
Definition: StmtOpenMP.h:3008
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:216
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
Definition: CGCall.h:356
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:59
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:67
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition: Decl.cpp:2501
Dataflow Directional Tag Classes.
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Expr * getPrevUpperBoundVariable() const
Definition: StmtOpenMP.h:867
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
This represents &#39;device&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:93
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:499
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
SourceRange getSourceRange(const SourceRange &Range)
Returns the SourceRange of a SourceRange.
Definition: FixIt.h:34
Expr * getCond() const
Definition: StmtOpenMP.h:781
This represents &#39;#pragma omp section&#39; directive.
Definition: StmtOpenMP.h:1282
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Definition: LangOptions.h:199
OpenMPScheduleClauseModifier M1
Definition: OpenMPKinds.h:126
This represents &#39;#pragma omp teams distribute&#39; directive.
Definition: StmtOpenMP.h:3418
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const Expr * getInit() const
Definition: Decl.h:1219
This represents &#39;#pragma omp simd&#39; directive.
Definition: StmtOpenMP.h:1007
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:70
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
DeclarationName - The name of a declaration.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
Definition: StmtOpenMP.h:1990
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
This represents clause &#39;linear&#39; in the &#39;#pragma omp ...&#39; directives.
SourceLocation getLocStart() const LLVM_READONLY
Definition: Decl.h:739
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; or &#39;OpaqueValueExp...
Definition: StmtOpenMP.h:2219
llvm::Module & getModule() const
This represents &#39;#pragma omp atomic&#39; directive.
Definition: StmtOpenMP.h:2122
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Expr * getCombinedInit() const
Definition: StmtOpenMP.h:903
const Expr * Replacement
Definition: ParsedAttr.h:67
unsigned getCollapsedNumber() const
Get number of collapsed loops.
Definition: StmtOpenMP.h:763
Expr * getCombinedNextLowerBound() const
Definition: StmtOpenMP.h:915
ArrayRef< Expr * > counters()
Definition: StmtOpenMP.h:939
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
SwitchStmt - This represents a &#39;switch&#39; stmt.
Definition: Stmt.h:1054
Expr * getCombinedNextUpperBound() const
Definition: StmtOpenMP.h:921
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1679
API for captured statement code generation.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
Complex values, per C99 6.2.5p11.
Definition: Type.h:2333
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
This file defines OpenMP AST classes for executable directives and clauses.
ArrayRef< Expr * > inits()
Definition: StmtOpenMP.h:951
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
Definition: Decl.h:1059
OpenMPScheduleClauseKind
OpenMP attributes for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:59
Base for LValueReferenceType and RValueReferenceType.
Definition: Type.h:2529
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
StringRef getMangledName(GlobalDecl GD)
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
Definition: Linkage.h:32
SourceLocation getLocStart() const LLVM_READONLY
Definition: Stmt.h:2317
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:445
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Definition: StmtOpenMP.h:195
Expr * getPrevLowerBoundVariable() const
Definition: StmtOpenMP.h:861
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:2052
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
Definition: CGStmt.cpp:38
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
Definition: ASTContext.h:2235
bool isLValueReferenceType() const
Definition: Type.h:6129
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:13820
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition: Stmt.cpp:1100
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
TranslationUnitDecl * getTranslationUnitDecl() const
Definition: ASTContext.h:997
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
This represents &#39;#pragma omp target parallel&#39; directive.
Definition: StmtOpenMP.h:2489
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
This represents &#39;nowait&#39; clause in the &#39;#pragma omp ...&#39; directive.
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
Definition: Type.cpp:1878
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
Definition: CGStmt.cpp:465
This represents &#39;num_tasks&#39; clause in the &#39;#pragma omp ...&#39; directive.
Privates[]
Gets the list of initial values for linear variables.
Definition: OpenMPClause.h:145
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:5916
Expr * getCombinedCond() const
Definition: StmtOpenMP.h:909
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Definition: CGExpr.cpp:1199
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition: Stmt.cpp:266
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1711
This represents &#39;#pragma omp taskloop simd&#39; directive.
Definition: StmtOpenMP.h:2874
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Expr * getPreCond() const
Definition: StmtOpenMP.h:777
This represents &#39;dist_schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
CGCapturedStmtInfo * CapturedStmtInfo
bool isGlobalReg() const
Definition: CGValue.h:256
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
const VariableArrayType * getAsVariableArrayType(QualType T) const
Definition: ASTContext.h:2398
const llvm::function_ref< void(CodeGenFunction &, llvm::Value *, const OMPTaskDataTy &)> TaskGenTy
This represents &#39;#pragma omp sections&#39; directive.
Definition: StmtOpenMP.h:1214
This represents &#39;#pragma omp target data&#39; directive.
Definition: StmtOpenMP.h:2314
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2793
capture_range captures()
Definition: Stmt.h:2260
A reference to a declared variable, function, enum, etc.
Definition: Expr.h:974
static RValue get(llvm::Value *V)
Definition: CGValue.h:86
bool isPointerType() const
Definition: Type.h:6113
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
QualType getType() const
Definition: Decl.h:648
An l-value expression is a reference to an object with independent storage.
Definition: Specifiers.h:114
A trivial tuple used to represent a source range.
SourceLocation getLocEnd() const LLVM_READONLY
Returns ending location of directive.
Definition: StmtOpenMP.h:171
LValue - This represents an lvalue references.
Definition: CGValue.h:167
This represents &#39;#pragma omp taskyield&#39; directive.
Definition: StmtOpenMP.h:1807
This represents &#39;#pragma omp distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3147
CanQualType BoolTy
Definition: ASTContext.h:1005
Represents a C array with a specified size that is not an integer-constant-expression.
Definition: Type.h:2827
This represents &#39;#pragma omp parallel sections&#39; directive.
Definition: StmtOpenMP.h:1668
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
SourceLocation getBegin() const
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition: Stmt.h:2309
llvm::Value * getPointer() const
Definition: CGValue.h:323
void EmitOMPTaskDirective(const OMPTaskDirective &S)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
Definition: DeclBase.cpp:882
bool isScalar() const
Definition: CGValue.h:52
This represents &#39;#pragma omp target parallel for&#39; directive.
Definition: StmtOpenMP.h:2549
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
Definition: DeclBase.h:419
This represents clause &#39;use_device_ptr&#39; in the &#39;#pragma omp ...&#39; directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1544
This represents &#39;#pragma omp taskloop&#39; directive.
Definition: StmtOpenMP.h:2809