22 #include "llvm/IR/CallSite.h" 23 using namespace clang;
24 using namespace CodeGen;
31 for (
const auto *C : S.
clauses()) {
33 if (
const auto *PreInit =
34 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
35 for (
const auto *I : PreInit->decls()) {
36 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
60 const bool EmitPreInitStmt =
true)
64 emitPreInitStmt(CGF, S);
65 if (!CapturedRegion.hasValue())
68 "Expected associated statement for inlined directive.");
70 for (
const auto &C : CS->captures()) {
71 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
72 auto *VD =
C.getCapturedVar();
74 "Canonical decl must be captured.");
76 const_cast<VarDecl *>(VD),
91 class OMPParallelScope final :
public OMPLexicalScope {
101 : OMPLexicalScope(CGF, S,
llvm::
None,
102 EmitPreInitStmt(S)) {}
107 class OMPTeamsScope final :
public OMPLexicalScope {
116 : OMPLexicalScope(CGF, S,
llvm::
None,
117 EmitPreInitStmt(S)) {}
125 for (
const auto *E : S.
counters()) {
126 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
130 (void)PreCondVars.
apply(CGF);
131 if (
const auto *PreInits = cast_or_null<DeclStmt>(S.
getPreInits())) {
132 for (
const auto *I : PreInits->decls())
141 emitPreInitStmt(CGF, S);
152 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
158 InlinedShareds(CGF) {
159 for (
const auto *C : S.
clauses()) {
161 if (
const auto *PreInit =
162 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
163 for (
const auto *I : PreInit->decls()) {
164 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
173 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
174 for (
const Expr *E : UDP->varlists()) {
175 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
176 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
183 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
184 if (
const Expr *E = TG->getReductionRef())
185 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
189 for (
auto &C : CS->captures()) {
190 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
191 auto *VD =
C.getCapturedVar();
193 "Canonical decl must be captured.");
195 isCapturedVar(CGF, VD) ||
218 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
219 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
220 OrigVD = OrigVD->getCanonicalDecl();
222 LambdaCaptureFields.lookup(OrigVD) ||
223 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
224 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
225 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
226 OrigDRE->getType(),
VK_LValue, OrigDRE->getExprLoc());
227 return EmitLValue(&DRE);
230 return EmitLValue(E);
237 if (SizeInChars.isZero()) {
242 Size = Size ? Builder.CreateNUWMul(Size, VlaSize.
NumElts)
246 if (SizeInChars.isZero())
247 return llvm::ConstantInt::get(SizeTy, 0);
248 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
250 return CGM.getSize(SizeInChars);
260 I != E; ++I, ++CurField, ++CurCap) {
261 if (CurField->hasCapturedVLAType()) {
264 CapturedVars.push_back(Val);
265 }
else if (CurCap->capturesThis()) {
266 CapturedVars.push_back(CXXThisValue);
267 }
else if (CurCap->capturesVariableByCopy()) {
268 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
272 if (!CurField->getType()->isAnyPointerType()) {
274 Address DstAddr = CreateMemTemp(
276 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
283 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
289 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
291 CapturedVars.push_back(CV);
293 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
294 CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
302 bool isReferenceType =
false) {
314 if (isReferenceType) {
333 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
335 if (!A->isVariablyModifiedType())
343 struct FunctionOptions {
348 const bool UIntPtrCastRequired =
true;
351 const bool RegisterCastedArgsOnly =
false;
353 const StringRef FunctionName;
354 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
355 bool RegisterCastedArgsOnly,
356 StringRef FunctionName)
357 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
358 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
359 FunctionName(FunctionName) {}
365 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
367 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
369 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
371 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
372 assert(CD->
hasBody() &&
"missing CapturedDecl body");
374 CXXThisValue =
nullptr;
384 auto I = FO.S->captures().begin();
386 if (!FO.UIntPtrCastRequired) {
405 if (FO.UIntPtrCastRequired &&
407 I->capturesVariableArrayType()))
410 if (I->capturesVariable() || I->capturesVariableByCopy()) {
411 CapVar = I->getCapturedVar();
413 }
else if (I->capturesThis()) {
416 assert(I->capturesVariableArrayType());
422 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
424 Ctx, DebugFunctionDecl,
425 CapVar ? CapVar->
getLocStart() : FD->getLocStart(),
426 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
432 Args.emplace_back(Arg);
434 TargetArgs.emplace_back(
435 FO.UIntPtrCastRequired
457 F->setDoesNotThrow();
458 F->setDoesNotRecurse();
464 I = FO.S->captures().begin();
468 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
476 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
477 const VarDecl *CurVD = I->getCapturedVar();
486 if (!FO.RegisterCastedArgsOnly)
487 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
495 if (FD->hasCapturedVLAType()) {
496 if (FO.UIntPtrCastRequired) {
499 Args[Cnt]->getName(), ArgLVal),
504 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
505 }
else if (I->capturesVariable()) {
506 const VarDecl *Var = I->getCapturedVar();
519 if (!FO.RegisterCastedArgsOnly) {
524 }
else if (I->capturesVariableByCopy()) {
525 assert(!FD->getType()->isAnyPointerType() &&
526 "Not expecting a captured pointer.");
527 const VarDecl *Var = I->getCapturedVar();
531 {Var, FO.UIntPtrCastRequired
533 FD->getType(), Args[Cnt]->getName(),
538 assert(I->capturesThis());
540 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress()}});
553 "CapturedStmtInfo should be set when generating the captured function");
556 bool NeedWrapperFunction =
560 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
561 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
563 llvm::raw_svector_ostream Out(Buffer);
564 Out << CapturedStmtInfo->getHelperName();
565 if (NeedWrapperFunction)
567 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
570 VLASizes, CXXThisValue, FO);
571 for (
const auto &LocalAddrPair : LocalAddrs) {
572 if (LocalAddrPair.second.first) {
573 setAddrOfLocalVar(LocalAddrPair.second.first,
574 LocalAddrPair.second.second);
577 for (
const auto &VLASizePair : VLASizes)
578 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
580 CapturedStmtInfo->EmitBody(*
this, CD->
getBody());
582 if (!NeedWrapperFunction)
585 FunctionOptions WrapperFO(&S,
true,
587 CapturedStmtInfo->getHelperName());
589 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
593 llvm::Function *WrapperF =
595 WrapperCGF.CXXThisValue, WrapperFO);
597 for (
const auto *Arg : Args) {
599 auto I = LocalAddrs.find(Arg);
600 if (I != LocalAddrs.end()) {
601 LValue LV = WrapperCGF.MakeAddrLValue(
603 I->second.first ? I->second.first->getType() : Arg->getType(),
605 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.
getLocStart());
607 auto EI = VLASizes.find(Arg);
608 if (EI != VLASizes.end()) {
609 CallArg = EI->second.second;
611 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
614 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.
getLocStart());
617 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
619 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.
getLocStart(),
621 WrapperCGF.FinishFunction();
636 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
637 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.
getElementType());
642 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
644 llvm::BasicBlock *BodyBB = createBasicBlock(
"omp.arraycpy.body");
645 llvm::BasicBlock *DoneBB = createBasicBlock(
"omp.arraycpy.done");
647 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
648 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
651 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
654 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
656 llvm::PHINode *SrcElementPHI =
657 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
658 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
663 llvm::PHINode *DestElementPHI =
664 Builder.CreatePHI(DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
665 DestElementPHI->addIncoming(DestBegin, EntryBB);
671 CopyGen(DestElementCurrent, SrcElementCurrent);
674 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
675 DestElementPHI, 1,
"omp.arraycpy.dest.element");
676 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
677 SrcElementPHI, 1,
"omp.arraycpy.src.element");
680 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
681 Builder.CreateCondBr(Done, DoneBB, BodyBB);
682 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
683 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
686 EmitBlock(DoneBB,
true);
694 if (BO && BO->getOpcode() == BO_Assign) {
696 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
697 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
698 EmitAggregateAssign(Dest, Src, OriginalType);
702 EmitOMPAggregateAssign(
703 DestAddr, SrcAddr, OriginalType,
704 [
this, Copy, SrcVD, DestVD](
Address DestElement,
Address SrcElement) {
709 Remap.
addPrivate(DestVD, [DestElement]() {
return DestElement; });
710 Remap.
addPrivate(SrcVD, [SrcElement]() {
return SrcElement; });
712 EmitIgnoredExpr(Copy);
718 Remap.
addPrivate(SrcVD, [SrcAddr]() {
return SrcAddr; });
719 Remap.
addPrivate(DestVD, [DestAddr]() {
return DestAddr; });
722 EmitIgnoredExpr(Copy);
728 if (!HaveInsertPoint())
730 bool FirstprivateIsLastprivate =
false;
733 for (
const auto *D : C->varlists())
742 bool MustEmitFirstprivateCopy =
743 CaptureRegions.size() == 1 && CaptureRegions.back() ==
OMPD_unknown;
745 auto IRef = C->varlist_begin();
746 auto InitsRef = C->inits().begin();
747 for (
const Expr *IInit : C->private_copies()) {
748 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
749 bool ThisFirstprivateIsLastprivate =
750 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
751 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
752 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
754 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
759 FirstprivateIsLastprivate =
760 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
761 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
762 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
764 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
769 LValue OriginalLVal = EmitLValue(&DRE);
776 OrigVD, [
this, VD, Type, OriginalLVal, VDInit]() {
779 if (!isa<CXXConstructExpr>(Init) ||
780 isTrivialInitializer(Init)) {
784 EmitAggregateAssign(Dest, OriginalLVal, Type);
786 EmitOMPAggregateAssign(
789 [
this, VDInit, Init](
Address DestElement,
795 setAddrOfLocalVar(VDInit, SrcElement);
796 EmitAnyExprToMem(Init, DestElement,
799 LocalDeclMap.erase(VDInit);
802 EmitAutoVarCleanups(Emission);
806 Address OriginalAddr = OriginalLVal.getAddress();
808 OrigVD, [
this, VDInit, OriginalAddr, VD]() {
812 setAddrOfLocalVar(VDInit, OriginalAddr);
814 LocalDeclMap.erase(VDInit);
815 return GetAddrOfLocalVar(VD);
818 assert(IsRegistered &&
819 "firstprivate var already registered as private");
827 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
833 if (!HaveInsertPoint())
837 auto IRef = C->varlist_begin();
838 for (
const Expr *IInit : C->private_copies()) {
839 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
840 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
841 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
842 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, [
this, VD]() {
845 return GetAddrOfLocalVar(VD);
847 assert(IsRegistered &&
"private var already registered as private");
857 if (!HaveInsertPoint())
864 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
866 auto IRef = C->varlist_begin();
867 auto ISrcRef = C->source_exprs().begin();
868 auto IDestRef = C->destination_exprs().begin();
869 for (
const Expr *AssignOp : C->assignment_ops()) {
870 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
877 if (getLangOpts().OpenMPUseTLS &&
878 getContext().getTargetInfo().isTLSSupported()) {
879 assert(CapturedStmtInfo->lookup(VD) &&
880 "Copyin threadprivates should have been captured!");
881 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
true, (*IRef)->getType(),
883 MasterAddr = EmitLValue(&DRE).getAddress();
884 LocalDeclMap.erase(VD);
888 : CGM.GetAddrOfGlobal(VD),
889 getContext().getDeclAlign(VD));
892 Address PrivateAddr = EmitLValue(*IRef).getAddress();
893 if (CopiedVars.size() == 1) {
896 CopyBegin = createBasicBlock(
"copyin.not.master");
897 CopyEnd = createBasicBlock(
"copyin.not.master.end");
898 Builder.CreateCondBr(
899 Builder.CreateICmpNE(
900 Builder.CreatePtrToInt(MasterAddr.
getPointer(), CGM.IntPtrTy),
901 Builder.CreatePtrToInt(PrivateAddr.getPointer(),
904 EmitBlock(CopyBegin);
907 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
909 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
910 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
919 EmitBlock(CopyEnd,
true);
927 if (!HaveInsertPoint())
929 bool HasAtLeastOneLastprivate =
false;
932 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
933 for (
const Expr *C : LoopDirective->counters()) {
940 HasAtLeastOneLastprivate =
true;
942 !getLangOpts().OpenMPSimd)
944 auto IRef = C->varlist_begin();
945 auto IDestRef = C->destination_exprs().begin();
946 for (
const Expr *IInit : C->private_copies()) {
949 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
952 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
954 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
955 PrivateScope.
addPrivate(DestVD, [
this, OrigVD, IRef]() {
957 const_cast<VarDecl *>(OrigVD),
958 CapturedStmtInfo->lookup(
960 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
961 return EmitLValue(&DRE).getAddress();
966 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
967 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
968 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, [
this, VD]() {
971 return GetAddrOfLocalVar(VD);
973 assert(IsRegistered &&
974 "lastprivate var already registered as private");
982 return HasAtLeastOneLastprivate;
988 if (!HaveInsertPoint())
996 llvm::BasicBlock *ThenBB =
nullptr;
997 llvm::BasicBlock *DoneBB =
nullptr;
998 if (IsLastIterCond) {
999 ThenBB = createBasicBlock(
".omp.lastprivate.then");
1000 DoneBB = createBasicBlock(
".omp.lastprivate.done");
1001 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1005 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1006 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1007 auto IC = LoopDirective->counters().begin();
1008 for (
const Expr *F : LoopDirective->finals()) {
1012 AlreadyEmittedVars.insert(D);
1014 LoopCountersAndUpdates[D] = F;
1019 auto IRef = C->varlist_begin();
1020 auto ISrcRef = C->source_exprs().begin();
1021 auto IDestRef = C->destination_exprs().begin();
1022 for (
const Expr *AssignOp : C->assignment_ops()) {
1023 const auto *PrivateVD =
1024 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1026 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1027 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1031 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1032 EmitIgnoredExpr(FinalExpr);
1034 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1035 const auto *DestVD =
1036 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1038 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1040 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1041 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1043 Address(Builder.CreateLoad(PrivateAddr),
1044 getNaturalTypeAlignment(RefTy->getPointeeType()));
1045 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1051 if (
const Expr *PostUpdate = C->getPostUpdateExpr())
1052 EmitIgnoredExpr(PostUpdate);
1055 EmitBlock(DoneBB,
true);
1061 if (!HaveInsertPoint())
1069 auto IPriv = C->privates().begin();
1070 auto IRed = C->reduction_ops().begin();
1071 auto ILHS = C->lhs_exprs().begin();
1072 auto IRHS = C->rhs_exprs().begin();
1073 for (
const Expr *Ref : C->varlists()) {
1074 Shareds.emplace_back(Ref);
1075 Privates.emplace_back(*IPriv);
1076 ReductionOps.emplace_back(*IRed);
1077 LHSs.emplace_back(*ILHS);
1078 RHSs.emplace_back(*IRHS);
1079 std::advance(IPriv, 1);
1080 std::advance(IRed, 1);
1081 std::advance(ILHS, 1);
1082 std::advance(IRHS, 1);
1087 auto ILHS = LHSs.begin();
1088 auto IRHS = RHSs.begin();
1089 auto IPriv = Privates.begin();
1090 for (
const Expr *IRef : Shareds) {
1091 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1093 RedCG.emitSharedLValue(*
this, Count);
1102 EmitAutoVarCleanups(Emission);
1106 RedCG.
getBaseDecl(Count), [BaseAddr]() {
return BaseAddr; });
1107 assert(IsRegistered &&
"private var already registered as private");
1111 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1112 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1114 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1115 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1118 PrivateScope.
addPrivate(LHSVD, [&RedCG, Count]() {
1122 RHSVD, [
this, PrivateVD]() {
return GetAddrOfLocalVar(PrivateVD); });
1123 }
else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1124 isa<ArraySubscriptExpr>(IRef)) {
1127 PrivateScope.
addPrivate(LHSVD, [&RedCG, Count]() {
1130 PrivateScope.
addPrivate(RHSVD, [
this, PrivateVD, RHSVD]() {
1131 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1132 ConvertTypeForMem(RHSVD->getType()),
1136 QualType Type = PrivateVD->getType();
1137 bool IsArray = getContext().getAsArrayType(Type) !=
nullptr;
1142 OriginalAddr = Builder.CreateElementBitCast(
1143 OriginalAddr, ConvertTypeForMem(LHSVD->getType()),
"lhs.begin");
1145 PrivateScope.
addPrivate(LHSVD, [OriginalAddr]() {
return OriginalAddr; });
1147 RHSVD, [
this, PrivateVD, RHSVD, IsArray]() {
1149 ? Builder.CreateElementBitCast(
1150 GetAddrOfLocalVar(PrivateVD),
1151 ConvertTypeForMem(RHSVD->getType()),
"rhs.begin")
1152 : GetAddrOfLocalVar(PrivateVD);
1164 if (!HaveInsertPoint())
1170 bool HasAtLeastOneReduction =
false;
1172 HasAtLeastOneReduction =
true;
1173 Privates.append(C->privates().begin(), C->privates().end());
1174 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1175 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1176 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1178 if (HasAtLeastOneReduction) {
1181 ReductionKind == OMPD_simd;
1182 bool SimpleReduction = ReductionKind == OMPD_simd;
1185 CGM.getOpenMPRuntime().emitReduction(
1187 {WithNowait, SimpleReduction, ReductionKind});
1196 llvm::BasicBlock *DoneBB =
nullptr;
1198 if (
const Expr *PostUpdate = C->getPostUpdateExpr()) {
1205 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1223 CodeGenBoundParametersTy;
1229 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1240 CGF, NumThreads, NumThreadsClause->getLocStart());
1245 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1247 const Expr *IfCond =
nullptr;
1250 C->getNameModifier() == OMPD_parallel) {
1251 IfCond = C->getCondition();
1256 OMPParallelScope
Scope(CGF, S);
1262 CodeGenBoundParameters(CGF, S, CapturedVars);
1265 CapturedVars, IfCond);
1304 EmitIgnoredExpr(UE);
1310 for (
const Expr *UE : C->updates())
1311 EmitIgnoredExpr(UE);
1316 JumpDest Continue = getJumpDestInCurrentScope(
"omp.body.continue");
1317 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1322 BreakContinueStack.pop_back();
1326 const Stmt &S,
bool RequiresCleanup,
const Expr *LoopCond,
1327 const Expr *IncExpr,
1330 auto LoopExit = getJumpDestInCurrentScope(
"omp.inner.for.end");
1333 auto CondBlock = createBasicBlock(
"omp.inner.for.cond");
1334 EmitBlock(CondBlock);
1336 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
1337 SourceLocToDebugLoc(R.
getEnd()));
1341 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
1342 if (RequiresCleanup)
1343 ExitBlock = createBasicBlock(
"omp.inner.for.cond.cleanup");
1345 llvm::BasicBlock *LoopBody = createBasicBlock(
"omp.inner.for.body");
1348 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1349 if (ExitBlock !=
LoopExit.getBlock()) {
1350 EmitBlock(ExitBlock);
1351 EmitBranchThroughCleanup(
LoopExit);
1354 EmitBlock(LoopBody);
1355 incrementProfileCounter(&S);
1358 JumpDest Continue = getJumpDestInCurrentScope(
"omp.inner.for.inc");
1359 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1365 EmitIgnoredExpr(IncExpr);
1367 BreakContinueStack.pop_back();
1368 EmitBranch(CondBlock);
1375 if (!HaveInsertPoint())
1378 bool HasLinears =
false;
1380 for (
const Expr *Init : C->inits()) {
1382 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1383 if (
const auto *Ref =
1386 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1388 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1394 EmitAutoVarCleanups(Emission);
1401 if (
const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1402 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1403 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1405 EmitIgnoredExpr(CS);
1414 if (!HaveInsertPoint())
1416 llvm::BasicBlock *DoneBB =
nullptr;
1419 auto IC = C->varlist_begin();
1420 for (
const Expr *F : C->finals()) {
1425 llvm::BasicBlock *ThenBB = createBasicBlock(
".omp.linear.pu");
1426 DoneBB = createBasicBlock(
".omp.linear.pu.done");
1427 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1431 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1433 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1434 (*IC)->getType(),
VK_LValue, (*IC)->getExprLoc());
1435 Address OrigAddr = EmitLValue(&DRE).getAddress();
1437 VarScope.
addPrivate(OrigVD, [OrigAddr]() {
return OrigAddr; });
1442 if (
const Expr *PostUpdate = C->getPostUpdateExpr())
1443 EmitIgnoredExpr(PostUpdate);
1446 EmitBlock(DoneBB,
true);
1454 unsigned ClauseAlignment = 0;
1455 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
1458 ClauseAlignment =
static_cast<unsigned>(AlignmentCI->getZExtValue());
1460 for (
const Expr *E : Clause->varlists()) {
1461 unsigned Alignment = ClauseAlignment;
1462 if (Alignment == 0) {
1469 E->getType()->getPointeeType()))
1472 assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1473 "alignment is not power of 2");
1474 if (Alignment != 0) {
1484 if (!HaveInsertPoint())
1488 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1489 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1492 EmitAutoVarCleanups(VarEmission);
1493 LocalDeclMap.erase(PrivateVD);
1494 (void)LoopScope.
addPrivate(VD, [&VarEmission]() {
1495 return VarEmission.getAllocatedAddress();
1497 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1499 (void)LoopScope.
addPrivate(PrivateVD, [
this, VD, E]() {
1500 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1501 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1502 E->getType(), VK_LValue, E->getExprLoc());
1503 return EmitLValue(&DRE).getAddress();
1506 (void)LoopScope.
addPrivate(PrivateVD, [&VarEmission]() {
1507 return VarEmission.getAllocatedAddress();
1514 if (!C->getNumForLoops())
1517 E = C->getLoopNumIterations().size();
1519 const auto *DRE = cast<DeclRefExpr>(C->getLoopCunter(I));
1520 const auto *VD = cast<VarDecl>(DRE->getDecl());
1522 if (LocalDeclMap.count(VD)) {
1523 (void)LoopScope.
addPrivate(VD, [
this, DRE, VD]() {
1524 return CreateMemTemp(DRE->getType(), VD->getName());
1532 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1533 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1551 if (!HaveInsertPoint())
1555 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1556 for (
const Expr *C : LoopDirective->counters()) {
1562 auto CurPrivate = C->privates().begin();
1563 for (
const Expr *E : C->varlists()) {
1564 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1565 const auto *PrivateVD =
1566 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1568 bool IsRegistered = PrivateScope.
addPrivate(VD, [
this, PrivateVD]() {
1570 EmitVarDecl(*PrivateVD);
1571 return GetAddrOfLocalVar(PrivateVD);
1573 assert(IsRegistered &&
"linear var already registered as private");
1577 EmitVarDecl(*PrivateVD);
1592 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1602 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1614 LoopStack.setParallel(!IsMonotonic);
1615 LoopStack.setVectorizeEnable();
1622 if (!HaveInsertPoint())
1624 llvm::BasicBlock *DoneBB =
nullptr;
1628 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1629 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1631 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1632 OrigVD->hasGlobalStorage() || CED) {
1637 llvm::BasicBlock *ThenBB = createBasicBlock(
".omp.final.then");
1638 DoneBB = createBasicBlock(
".omp.final.done");
1639 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1645 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1649 (*IPC)->getType(),
VK_LValue, (*IPC)->getExprLoc());
1650 OrigAddr = EmitLValue(&DRE).getAddress();
1653 VarScope.
addPrivate(OrigVD, [OrigAddr]() {
return OrigAddr; });
1661 EmitBlock(DoneBB,
true);
1674 auto VDecl = cast<VarDecl>(Helper->
getDecl());
1683 "Expected simd directive");
1684 OMPLoopScope PreInitScope(CGF, S);
1701 llvm::BasicBlock *ContBlock =
nullptr;
1716 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1724 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1750 if (HasLastprivateClause)
1769 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
1772 void CodeGenFunction::EmitOMPOuterLoop(
1775 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1781 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
1787 llvm::BasicBlock *CondBlock = createBasicBlock(
"omp.dispatch.cond");
1788 EmitBlock(CondBlock);
1790 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
1791 SourceLocToDebugLoc(R.
getEnd()));
1794 if (!DynamicOrOrdered) {
1798 EmitIgnoredExpr(LoopArgs.EUB);
1800 EmitIgnoredExpr(LoopArgs.Init);
1802 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1805 RT.emitForNext(*
this, S.
getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1806 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1811 llvm::BasicBlock *ExitBlock = LoopExit.
getBlock();
1813 ExitBlock = createBasicBlock(
"omp.dispatch.cleanup");
1815 llvm::BasicBlock *LoopBody = createBasicBlock(
"omp.dispatch.body");
1816 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1817 if (ExitBlock != LoopExit.
getBlock()) {
1818 EmitBlock(ExitBlock);
1819 EmitBranchThroughCleanup(LoopExit);
1821 EmitBlock(LoopBody);
1825 if (DynamicOrOrdered)
1826 EmitIgnoredExpr(LoopArgs.Init);
1829 JumpDest Continue = getJumpDestInCurrentScope(
"omp.dispatch.inc");
1830 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1835 LoopStack.setParallel(!IsMonotonic);
1837 EmitOMPSimdInit(S, IsMonotonic);
1849 CodeGenLoop(CGF, S, LoopExit);
1852 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1856 BreakContinueStack.pop_back();
1857 if (!DynamicOrOrdered) {
1859 EmitIgnoredExpr(LoopArgs.NextLB);
1860 EmitIgnoredExpr(LoopArgs.NextUB);
1863 EmitBranch(CondBlock);
1866 EmitBlock(LoopExit.getBlock());
1870 if (!DynamicOrOrdered)
1877 void CodeGenFunction::EmitOMPForOuterLoop(
1880 const OMPLoopArguments &LoopArgs,
1885 const bool DynamicOrOrdered =
1886 Ordered || RT.isDynamic(ScheduleKind.
Schedule);
1889 !RT.isStaticNonchunked(ScheduleKind.
Schedule,
1890 LoopArgs.Chunk !=
nullptr)) &&
1891 "static non-chunked schedule does not need outer loop");
1944 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
1947 if (DynamicOrOrdered) {
1948 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1949 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
1952 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1954 RT.emitForDispatchInit(*
this, S.
getLocStart(), ScheduleKind, IVSize,
1955 IVSigned, Ordered, DipatchRTInputValues);
1957 CGOpenMPRuntime::StaticRTInput StaticInit(
1958 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1959 LoopArgs.ST, LoopArgs.Chunk);
1961 ScheduleKind, StaticInit);
1965 const unsigned IVSize,
1966 const bool IVSigned) {
1973 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1974 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1975 OuterLoopArgs.IncExpr = S.
getInc();
1976 OuterLoopArgs.Init = S.
getInit();
1977 OuterLoopArgs.Cond = S.
getCond();
1980 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1985 const unsigned IVSize,
const bool IVSigned) {}
1987 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1989 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
1990 const CodeGenLoopTy &CodeGenLoopContent) {
2000 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
2003 CGOpenMPRuntime::StaticRTInput StaticInit(
2004 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
2005 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2006 RT.emitDistributeStaticInit(*
this, S.
getLocStart(), ScheduleKind, StaticInit);
2019 OMPLoopArguments OuterLoopArgs;
2020 OuterLoopArgs.LB = LoopArgs.LB;
2021 OuterLoopArgs.UB = LoopArgs.UB;
2022 OuterLoopArgs.ST = LoopArgs.ST;
2023 OuterLoopArgs.IL = LoopArgs.IL;
2024 OuterLoopArgs.Chunk = LoopArgs.Chunk;
2028 OuterLoopArgs.IncExpr = IncExpr;
2042 EmitOMPOuterLoop(
false,
false, S,
2043 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2047 static std::pair<LValue, LValue>
2090 static std::pair<llvm::Value *, llvm::Value *>
2100 QualType IteratorTy = IVExpr->getType();
2105 return {LBVal, UBVal};
2111 const auto &Dir = cast<OMPLoopDirective>(S);
2113 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2116 CapturedVars.push_back(LBCast);
2118 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2122 CapturedVars.push_back(UBCast);
2132 bool HasCancel =
false;
2134 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2135 HasCancel = D->hasCancel();
2136 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2137 HasCancel = D->hasCancel();
2138 else if (
const auto *D =
2139 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2140 HasCancel = D->hasCancel();
2152 CGInlinedWorksharingLoop,
2162 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
2163 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
2172 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
2173 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
2182 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2192 llvm::Constant *Addr;
2195 S, ParentName, Fn, Addr,
true, CodeGen);
2196 assert(Fn && Addr &&
"Target device function emission failed.");
2208 struct ScheduleKindModifiersTy {
2215 :
Kind(Kind), M1(M1), M2(M2) {}
2225 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2226 EmitVarDecl(*IVDecl);
2232 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2239 bool HasLastprivateClause;
2242 OMPLoopScope PreInitScope(*
this, S);
2247 llvm::BasicBlock *ContBlock =
nullptr;
2248 if (ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
2252 llvm::BasicBlock *ThenBlock = createBasicBlock(
"omp.precond.then");
2253 ContBlock = createBasicBlock(
"omp.precond.end");
2255 getProfileCount(&S));
2256 EmitBlock(ThenBlock);
2257 incrementProfileCounter(&S);
2261 bool Ordered =
false;
2263 if (OrderedClause->getNumForLoops())
2264 RT.emitDoacrossInit(*
this, S, OrderedClause->getLoopNumIterations());
2271 bool HasLinears = EmitOMPLinearClauseInit(S);
2274 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
2275 LValue LB = Bounds.first;
2276 LValue UB = Bounds.second;
2285 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2289 CGM.getOpenMPRuntime().emitBarrierCall(
2293 EmitOMPPrivateClause(S, LoopScope);
2294 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2295 EmitOMPReductionClauseInit(S, LoopScope);
2296 EmitOMPPrivateLoopCounters(S, LoopScope);
2297 EmitOMPLinearClause(S, LoopScope);
2304 ScheduleKind.
Schedule = C->getScheduleKind();
2305 ScheduleKind.
M1 = C->getFirstScheduleModifier();
2306 ScheduleKind.
M2 = C->getSecondScheduleModifier();
2307 if (
const Expr *Ch = C->getChunkSize()) {
2308 Chunk = EmitScalarExpr(Ch);
2309 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2314 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2315 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2320 if (RT.isStaticNonchunked(ScheduleKind.
Schedule,
2321 Chunk !=
nullptr) &&
2324 EmitOMPSimdInit(S,
true);
2330 CGOpenMPRuntime::StaticRTInput StaticInit(
2334 ScheduleKind, StaticInit);
2336 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
2357 const bool IsMonotonic =
2358 Ordered || ScheduleKind.
Schedule == OMPC_SCHEDULE_static ||
2360 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2361 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2367 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2368 LoopArguments, CGDispatchBounds);
2373 return CGF.
Builder.CreateIsNotNull(
2377 EmitOMPReductionClauseFinal(
2379 ? OMPD_parallel_for_simd
2384 return CGF.
Builder.CreateIsNotNull(
2388 if (HasLastprivateClause)
2389 EmitOMPLastprivateClauseFinal(
2391 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.
getLocStart())));
2394 return CGF.
Builder.CreateIsNotNull(
2400 EmitBranch(ContBlock);
2401 EmitBlock(ContBlock,
true);
2404 return HasLastprivateClause;
2410 static std::pair<LValue, LValue>
2412 const auto &LS = cast<OMPLoopDirective>(S);
2424 static std::pair<llvm::Value *, llvm::Value *>
2427 const auto &LS = cast<OMPLoopDirective>(S);
2428 const Expr *IVExpr = LS.getIterationVariable();
2432 return {LBVal, UBVal};
2436 bool HasLastprivates =
false;
2446 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_for, CodeGen,
2452 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(), OMPD_for);
2456 bool HasLastprivates =
false;
2465 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2470 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(), OMPD_for);
2485 bool HasLastprivates =
false;
2486 auto &&CodeGen = [&S, CapturedStmt, CS,
2493 CGF.Builder.getInt32(0));
2494 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr 2495 ? CGF.Builder.getInt32(CS->size() - 1)
2496 : CGF.Builder.getInt32(0);
2500 CGF.Builder.getInt32(1));
2502 CGF.Builder.getInt32(0));
2527 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
2529 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.
getLocStart()),
2530 ExitBB, CS ==
nullptr ? 1 : CS->size());
2532 unsigned CaseNumber = 0;
2534 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2535 CGF.EmitBlock(CaseBB);
2536 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2537 CGF.EmitStmt(SubStmt);
2538 CGF.EmitBranch(ExitBB);
2542 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2543 CGF.EmitBlock(CaseBB);
2544 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2545 CGF.EmitStmt(CapturedStmt);
2546 CGF.EmitBranch(ExitBB);
2548 CGF.EmitBlock(ExitBB,
true);
2552 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2556 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2560 CGF.EmitOMPPrivateClause(S, LoopScope);
2561 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2562 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2567 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
2568 CGOpenMPRuntime::StaticRTInput StaticInit(
2569 32,
true,
false, IL.getAddress(),
2571 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2572 CGF, S.
getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2575 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2576 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2577 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2579 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.
getLocStart()), IV);
2581 CGF.EmitOMPInnerLoop(S,
false, &Cond, &Inc, BodyGen,
2585 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.
getLocEnd(),
2586 S.getDirectiveKind());
2588 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2589 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
2592 return CGF.
Builder.CreateIsNotNull(
2597 if (HasLastprivates)
2604 bool HasCancel =
false;
2605 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2606 HasCancel = OSD->hasCancel();
2607 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2608 HasCancel = OPSD->hasCancel();
2610 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_sections, CodeGen,
2618 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(),
2630 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(),
2640 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_section, CodeGen,
2654 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2655 DestExprs.append(C->destination_exprs().begin(),
2656 C->destination_exprs().end());
2657 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2658 AssignmentOps.append(C->assignment_ops().begin(),
2659 C->assignment_ops().end());
2672 CGM.getOpenMPRuntime().emitSingleRegion(*
this, CodeGen, S.
getLocStart(),
2673 CopyprivateVars, DestExprs,
2674 SrcExprs, AssignmentOps);
2679 CGM.getOpenMPRuntime().emitBarrierCall(
2691 CGM.getOpenMPRuntime().emitMasterRegion(*
this, CodeGen, S.
getLocStart());
2699 const Expr *Hint =
nullptr;
2701 Hint = HintClause->getHint();
2703 CGM.getOpenMPRuntime().emitCriticalRegion(*
this,
2741 CGF.EmitSections(S);
2754 auto PartId = std::next(I);
2755 auto TaskT = std::next(I, 4);
2760 const Expr *Cond = Clause->getCondition();
2762 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2763 Data.
Final.setInt(CondConstant);
2765 Data.
Final.setPointer(EvaluateExprAsBool(Cond));
2768 Data.
Final.setInt(
false);
2772 const Expr *Prio = Clause->getPriority();
2774 Data.
Priority.setPointer(EmitScalarConversion(
2775 EmitScalarExpr(Prio), Prio->
getType(),
2776 getContext().getIntTypeForBitwidth(32, 1),
2784 auto IRef = C->varlist_begin();
2785 for (
const Expr *IInit : C->private_copies()) {
2786 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2787 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2794 EmittedAsPrivate.clear();
2797 auto IRef = C->varlist_begin();
2798 auto IElemInitRef = C->inits().begin();
2799 for (
const Expr *IInit : C->private_copies()) {
2800 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2801 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2811 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2813 auto IRef = C->varlist_begin();
2814 auto ID = C->destination_exprs().begin();
2815 for (
const Expr *IInit : C->private_copies()) {
2816 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2817 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2821 LastprivateDstsOrigs.insert(
2822 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2823 cast<DeclRefExpr>(*IRef)});
2831 auto IPriv = C->privates().begin();
2832 auto IRed = C->reduction_ops().begin();
2833 auto ILHS = C->lhs_exprs().begin();
2834 auto IRHS = C->rhs_exprs().begin();
2835 for (
const Expr *Ref : C->varlists()) {
2839 LHSs.emplace_back(*ILHS);
2840 RHSs.emplace_back(*IRHS);
2841 std::advance(IPriv, 1);
2842 std::advance(IRed, 1);
2843 std::advance(ILHS, 1);
2844 std::advance(IRHS, 1);
2847 Data.
Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2851 for (
const Expr *IRef : C->varlists())
2852 Data.
Dependences.emplace_back(C->getDependencyKind(), IRef);
2853 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2860 enum { PrivatesParam = 2, CopyFnParam = 3 };
2862 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2863 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2864 CS->getCapturedDecl()->getParam(PrivatesParam)));
2868 CallArgs.push_back(PrivatesPtr);
2870 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2871 Address PrivatePtr = CGF.CreateMemTemp(
2872 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
2873 PrivatePtrs.emplace_back(VD, PrivatePtr);
2874 CallArgs.push_back(PrivatePtr.getPointer());
2877 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2879 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
2880 ".firstpriv.ptr.addr");
2881 PrivatePtrs.emplace_back(VD, PrivatePtr);
2882 CallArgs.push_back(PrivatePtr.getPointer());
2885 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2887 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
2888 ".lastpriv.ptr.addr");
2889 PrivatePtrs.emplace_back(VD, PrivatePtr);
2890 CallArgs.push_back(PrivatePtr.getPointer());
2892 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2894 for (
const auto &Pair : LastprivateDstsOrigs) {
2895 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2897 const_cast<VarDecl *>(OrigVD),
2898 CGF.CapturedStmtInfo->lookup(
2900 Pair.second->getType(),
VK_LValue, Pair.second->getExprLoc());
2901 Scope.
addPrivate(Pair.first, [&CGF, &DRE]() {
2902 return CGF.EmitLValue(&DRE).getAddress();
2905 for (
const auto &Pair : PrivatePtrs) {
2907 CGF.getContext().getDeclAlign(Pair.first));
2912 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2915 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2916 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2917 for (
unsigned Cnt = 0, E = Data.
ReductionVars.size(); Cnt < E; ++Cnt) {
2918 RedCG.emitSharedLValue(CGF, Cnt);
2923 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2928 Address(CGF.EmitScalarConversion(
2929 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
2930 CGF.getContext().getPointerType(
2946 auto IPriv = C->privates().begin();
2947 auto IRed = C->reduction_ops().begin();
2948 auto ITD = C->taskgroup_descriptors().begin();
2949 for (
const Expr *Ref : C->varlists()) {
2950 InRedVars.emplace_back(Ref);
2951 InRedPrivs.emplace_back(*IPriv);
2952 InRedOps.emplace_back(*IRed);
2953 TaskgroupDescriptors.emplace_back(*ITD);
2954 std::advance(IPriv, 1);
2955 std::advance(IRed, 1);
2956 std::advance(ITD, 1);
2962 if (!InRedVars.empty()) {
2964 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2965 RedCG.emitSharedLValue(CGF, Cnt);
2972 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2975 CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
2976 TaskgroupDescriptors[Cnt]->getExprLoc());
2980 CGF.EmitScalarConversion(
2981 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
2982 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2983 InRedPrivs[Cnt]->getExprLoc()),
2995 llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2996 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.
Tied,
2998 OMPLexicalScope
Scope(*
this, S);
2999 TaskGen(*
this, OutlinedFn, Data);
3037 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3040 auto PartId = std::next(I);
3041 auto TaskT = std::next(I, 4);
3044 Data.
Final.setInt(
false);
3047 auto IRef = C->varlist_begin();
3048 auto IElemInitRef = C->inits().begin();
3049 for (
auto *IInit : C->private_copies()) {
3063 getContext(), getContext().getTranslationUnitDecl(), 0);
3065 QualType BaseAndPointersType = getContext().getConstantArrayType(
3069 getContext(), Data, BaseAndPointersType, CD, S.
getLocStart());
3071 getContext(), Data, BaseAndPointersType, CD, S.
getLocStart());
3072 QualType SizesType = getContext().getConstantArrayType(
3082 [&InputInfo]() {
return InputInfo.
SizesArray; });
3087 for (
const Expr *IRef : C->varlists())
3088 Data.
Dependences.emplace_back(C->getDependencyKind(), IRef);
3089 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3094 enum { PrivatesParam = 2, CopyFnParam = 3 };
3096 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3097 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3098 CS->getCapturedDecl()->getParam(PrivatesParam)));
3102 CallArgs.push_back(PrivatesPtr);
3104 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3106 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
3107 ".firstpriv.ptr.addr");
3108 PrivatePtrs.emplace_back(VD, PrivatePtr);
3109 CallArgs.push_back(PrivatePtr.getPointer());
3111 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3113 for (
const auto &Pair : PrivatePtrs) {
3115 CGF.getContext().getDeclAlign(Pair.first));
3123 CGF.GetAddrOfLocalVar(BPVD), 0, CGF.getPointerSize());
3125 CGF.GetAddrOfLocalVar(PVD), 0, CGF.getPointerSize());
3126 InputInfo.
SizesArray = CGF.Builder.CreateConstArrayGEP(
3127 CGF.GetAddrOfLocalVar(SVD), 0, CGF.getSizeSize());
3131 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
3134 llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3135 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
true,
3137 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<
OMPNowaitClause>() ? 1 : 0);
3139 getContext().getIntTypeForBitwidth(32, 0),
3142 CGM.getOpenMPRuntime().emitTaskCall(*
this, S.getLocStart(), S, OutlinedFn,
3143 SharedsTy, CapturedStruct, &IfCond, Data);
3149 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3151 const Expr *IfCond =
nullptr;
3154 C->getNameModifier() == OMPD_task) {
3155 IfCond = C->getCondition();
3166 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3169 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.
getLocStart(), S, OutlinedFn,
3170 SharedsTy, CapturedStruct, IfCond,
3173 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3178 CGM.getOpenMPRuntime().emitTaskyieldCall(*
this, S.
getLocStart());
3182 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(), OMPD_barrier);
3186 CGM.getOpenMPRuntime().emitTaskwaitCall(*
this, S.
getLocStart());
3198 auto IPriv = C->privates().begin();
3199 auto IRed = C->reduction_ops().begin();
3200 auto ILHS = C->lhs_exprs().begin();
3201 auto IRHS = C->rhs_exprs().begin();
3202 for (
const Expr *Ref : C->varlists()) {
3206 LHSs.emplace_back(*ILHS);
3207 RHSs.emplace_back(*IRHS);
3208 std::advance(IPriv, 1);
3209 std::advance(IRed, 1);
3210 std::advance(ILHS, 1);
3211 std::advance(IRHS, 1);
3217 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3220 false, E->getType());
3225 CGM.getOpenMPRuntime().emitTaskgroupRegion(*
this, CodeGen, S.
getLocStart());
3229 CGM.getOpenMPRuntime().emitFlush(
3233 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3234 FlushClause->varlist_end());
3245 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3246 EmitVarDecl(*IVDecl);
3252 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3259 bool HasLastprivateClause =
false;
3262 OMPLoopScope PreInitScope(*
this, S);
3267 llvm::BasicBlock *ContBlock =
nullptr;
3268 if (ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
3272 llvm::BasicBlock *ThenBlock = createBasicBlock(
"omp.precond.then");
3273 ContBlock = createBasicBlock(
"omp.precond.end");
3275 getProfileCount(&S));
3276 EmitBlock(ThenBlock);
3277 incrementProfileCounter(&S);
3286 *
this, cast<DeclRefExpr>(
3291 *
this, cast<DeclRefExpr>(
3301 if (EmitOMPFirstprivateClause(S, LoopScope)) {
3305 CGM.getOpenMPRuntime().emitBarrierCall(
3309 EmitOMPPrivateClause(S, LoopScope);
3313 EmitOMPReductionClauseInit(S, LoopScope);
3314 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3315 EmitOMPPrivateLoopCounters(S, LoopScope);
3322 ScheduleKind = C->getDistScheduleKind();
3323 if (
const Expr *Ch = C->getChunkSize()) {
3324 Chunk = EmitScalarExpr(Ch);
3325 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3330 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3331 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3341 if (RT.isStaticNonchunked(ScheduleKind,
3342 Chunk !=
nullptr)) {
3344 EmitOMPSimdInit(S,
true);
3345 CGOpenMPRuntime::StaticRTInput StaticInit(
3347 LB.
getAddress(), UB.getAddress(), ST.getAddress());
3348 RT.emitDistributeStaticInit(*
this, S.
getLocStart(), ScheduleKind,
3351 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
3372 CodeGenLoop(CGF, S, LoopExit);
3375 EmitBlock(LoopExit.getBlock());
3377 RT.emitForStaticFinish(*
this, S.getLocStart(), S.getDirectiveKind());
3381 const OMPLoopArguments LoopArguments = {
3384 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3389 return CGF.
Builder.CreateIsNotNull(
3399 ReductionKind = OMPD_parallel_for_simd;
3401 ReductionKind = OMPD_parallel_for;
3403 ReductionKind = OMPD_simd;
3407 "No reduction clauses is allowed in distribute directive.");
3409 EmitOMPReductionClauseFinal(S, ReductionKind);
3413 return CGF.
Builder.CreateIsNotNull(
3418 if (HasLastprivateClause) {
3419 EmitOMPLastprivateClauseFinal(
3421 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.
getLocStart())));
3427 EmitBranch(ContBlock);
3428 EmitBlock(ContBlock,
true);
3439 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
3448 Fn->setDoesNotRecurse();
3455 "No associated statement must be in ordered depend construct.");
3457 CGM.getOpenMPRuntime().emitDoacrossOrdered(*
this, DC);
3466 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3468 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.
getLocStart(),
3469 OutlinedFn, CapturedVars);
3476 CGM.getOpenMPRuntime().emitOrderedRegion(*
this, CodeGen, S.
getLocStart(), !C);
3483 "DestType must have scalar evaluation kind.");
3484 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
3495 "DestType must have complex evaluation kind.");
3504 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3506 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
3511 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
3513 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
3524 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3525 : llvm::AtomicOrdering::Monotonic,
3532 switch (getEvaluationKind(LVal.
getType())) {
3535 *
this, RVal, RValTy, LVal.
getType(), Loc)),
3544 llvm_unreachable(
"Must be a scalar or complex.");
3552 assert(V->
isLValue() &&
"V of 'omp atomic read' is not lvalue");
3553 assert(X->
isLValue() &&
"X of 'omp atomic read' is not lvalue");
3556 RValue Res = XLValue.isGlobalReg()
3560 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3561 : llvm::AtomicOrdering::Monotonic,
3562 XLValue.isVolatile());
3576 assert(X->
isLValue() &&
"X of 'omp atomic write' is not lvalue");
3589 llvm::AtomicOrdering AO,
3590 bool IsXLHSInRHSPart) {
3595 if (BO == BO_Comma || !Update.
isScalar() ||
3603 return std::make_pair(
false,
RValue::get(
nullptr));
3605 llvm::AtomicRMWInst::BinOp RMWOp;
3608 RMWOp = llvm::AtomicRMWInst::Add;
3611 if (!IsXLHSInRHSPart)
3612 return std::make_pair(
false,
RValue::get(
nullptr));
3613 RMWOp = llvm::AtomicRMWInst::Sub;
3619 RMWOp = llvm::AtomicRMWInst::Or;
3622 RMWOp = llvm::AtomicRMWInst::Xor;
3626 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3627 : llvm::AtomicRMWInst::Max)
3628 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3629 : llvm::AtomicRMWInst::UMax);
3633 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3634 : llvm::AtomicRMWInst::Min)
3635 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3636 : llvm::AtomicRMWInst::UMin);
3639 RMWOp = llvm::AtomicRMWInst::Xchg;
3648 return std::make_pair(
false,
RValue::get(
nullptr));
3667 llvm_unreachable(
"Unsupported atomic update operation");
3670 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3671 UpdateVal = CGF.
Builder.CreateIntCast(
3695 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3706 const Expr *UE,
bool IsXLHSInRHSPart,
3709 "Update expr in 'atomic update' must be a binary operator.");
3717 assert(X->
isLValue() &&
"X of 'omp atomic update' is not lvalue");
3720 llvm::AtomicOrdering AO = IsSeqCst
3721 ? llvm::AtomicOrdering::SequentiallyConsistent
3722 : llvm::AtomicOrdering::Monotonic;
3723 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3724 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3727 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) {
3733 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3756 llvm_unreachable(
"Must be a scalar or complex.");
3760 bool IsPostfixUpdate,
const Expr *V,
3762 const Expr *UE,
bool IsXLHSInRHSPart,
3764 assert(X->
isLValue() &&
"X of 'omp atomic capture' is not lvalue");
3765 assert(V->
isLValue() &&
"V of 'omp atomic capture' is not lvalue");
3770 llvm::AtomicOrdering AO = IsSeqCst
3771 ? llvm::AtomicOrdering::SequentiallyConsistent
3772 : llvm::AtomicOrdering::Monotonic;
3777 "Update expr in 'atomic capture' must be a binary operator.");
3785 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3786 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3788 NewVValType = XRValExpr->getType();
3790 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3791 IsPostfixUpdate](
RValue XRValue) {
3795 NewVVal = IsPostfixUpdate ? XRValue : Res;
3799 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3802 if (IsPostfixUpdate) {
3804 NewVVal = Res.second;
3818 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) {
3824 XLValue, ExprRValue, BO_Assign,
false, AO,
3828 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3842 bool IsSeqCst,
bool IsPostfixUpdate,
3844 const Expr *UE,
bool IsXLHSInRHSPart,
3859 IsXLHSInRHSPart, Loc);
3863 case OMPC_num_threads:
3865 case OMPC_firstprivate:
3866 case OMPC_lastprivate:
3867 case OMPC_reduction:
3868 case OMPC_task_reduction:
3869 case OMPC_in_reduction:
3879 case OMPC_copyprivate:
3881 case OMPC_proc_bind:
3888 case OMPC_mergeable:
3893 case OMPC_num_teams:
3894 case OMPC_thread_limit:
3896 case OMPC_grainsize:
3898 case OMPC_num_tasks:
3900 case OMPC_dist_schedule:
3901 case OMPC_defaultmap:
3905 case OMPC_use_device_ptr:
3906 case OMPC_is_device_ptr:
3907 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
3916 if (C->getClauseKind() != OMPC_seq_cst) {
3917 Kind = C->getClauseKind();
3923 if (
const auto *EWC = dyn_cast<ExprWithCleanups>(CS))
3924 enterFullExpression(EWC);
3926 if (
const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3927 for (
const Stmt *C : Compound->body()) {
3928 if (
const auto *EWC = dyn_cast<ExprWithCleanups>(C))
3929 enterFullExpression(EWC);
3935 CGF.EmitStopPoint(CS);
3941 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_atomic, CodeGen);
3952 OMPLexicalScope
Scope(CGF, S, OMPD_target);
3960 llvm::Function *Fn =
nullptr;
3961 llvm::Constant *FnID =
nullptr;
3963 const Expr *IfCond =
nullptr;
3967 C->getNameModifier() == OMPD_target) {
3968 IfCond = C->getCondition();
3974 const Expr *Device =
nullptr;
3976 Device = C->getDevice();
3981 bool IsOffloadEntry =
true;
3985 IsOffloadEntry =
false;
3988 IsOffloadEntry =
false;
3990 assert(CGF.
CurFuncDecl &&
"No parent declaration for target region!");
3991 StringRef ParentName;
3994 if (
const auto *D = dyn_cast<CXXConstructorDecl>(CGF.
CurFuncDecl))
3996 else if (
const auto *D = dyn_cast<CXXDestructorDecl>(CGF.
CurFuncDecl))
4004 IsOffloadEntry, CodeGen);
4005 OMPLexicalScope
Scope(CGF, S, OMPD_task);
4021 StringRef ParentName,
4027 llvm::Constant *Addr;
4030 S, ParentName, Fn, Addr,
true, CodeGen);
4031 assert(Fn && Addr &&
"Target device function emission failed.");
4053 const Expr *NumTeams = NT ? NT->getNumTeams() :
nullptr;
4054 const Expr *ThreadLimit = TL ? TL->getThreadLimit() :
nullptr;
4060 OMPTeamsScope
Scope(CGF, S);
4092 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4093 CGF.EmitOMPPrivateClause(S, PrivateScope);
4094 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4096 CGF.EmitStmt(CS->getCapturedStmt());
4097 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4111 llvm::Constant *Addr;
4114 S, ParentName, Fn, Addr,
true, CodeGen);
4115 assert(Fn && Addr &&
"Target device function emission failed.");
4139 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4141 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4143 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4157 llvm::Constant *Addr;
4160 S, ParentName, Fn, Addr,
true, CodeGen);
4161 assert(Fn && Addr &&
"Target device function emission failed.");
4185 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4187 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4189 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4203 llvm::Constant *Addr;
4206 S, ParentName, Fn, Addr,
true, CodeGen);
4207 assert(Fn && Addr &&
"Target device function emission failed.");
4230 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4232 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4234 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4252 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4254 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4256 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4275 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4277 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4279 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4298 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4300 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4301 CGF, OMPD_distribute, CodeGenDistribute,
false);
4302 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4323 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4325 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4326 CGF, OMPD_distribute, CodeGenDistribute,
false);
4327 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4345 llvm::Constant *Addr;
4348 S, ParentName, Fn, Addr,
true, CodeGen);
4349 assert(Fn && Addr &&
"Target device function emission failed.");
4375 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4377 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4378 CGF, OMPD_distribute, CodeGenDistribute,
false);
4379 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4397 llvm::Constant *Addr;
4400 S, ParentName, Fn, Addr,
true, CodeGen);
4401 assert(Fn && Addr &&
"Target device function emission failed.");
4414 CGM.getOpenMPRuntime().emitCancellationPointCall(*
this, S.
getLocStart(),
4419 const Expr *IfCond =
nullptr;
4422 C->getNameModifier() == OMPD_cancel) {
4423 IfCond = C->getCondition();
4427 CGM.getOpenMPRuntime().emitCancelCall(*
this, S.
getLocStart(), IfCond,
4433 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4434 Kind == OMPD_target_parallel)
4436 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4437 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4438 Kind == OMPD_distribute_parallel_for ||
4439 Kind == OMPD_target_parallel_for ||
4440 Kind == OMPD_teams_distribute_parallel_for ||
4441 Kind == OMPD_target_teams_distribute_parallel_for);
4442 return OMPCancelStack.getExitBlock();
4447 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4448 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4449 auto OrigVarIt = C.varlist_begin();
4450 auto InitIt = C.inits().begin();
4451 for (
const Expr *PvtVarIt : C.private_copies()) {
4452 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4453 const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4454 const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4460 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4463 const auto *ME = cast<MemberExpr>(OED->getInit());
4464 assert(isa<CXXThisExpr>(ME->getBase()) &&
4465 "Base should be the current struct!");
4466 MatchingVD = ME->getMemberDecl();
4471 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4472 if (InitAddrIt == CaptureDeviceAddrMap.end())
4475 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, [
this, OrigVD,
4484 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4485 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4486 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4487 setAddrOfLocalVar(InitVD, InitAddr);
4495 LocalDeclMap.erase(InitVD);
4498 return GetAddrOfLocalVar(PvtVD);
4500 assert(IsRegistered &&
"firstprivate var already registered as private");
4512 CGOpenMPRuntime::TargetDataInfo Info(
true);
4517 bool PrivatizeDevicePointers =
false;
4519 bool &PrivatizeDevicePointers;
4522 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
4523 :
PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4525 PrivatizeDevicePointers =
true;
4528 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4530 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4537 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4541 PrivatizeDevicePointers =
false;
4547 if (PrivatizeDevicePointers) {
4551 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4552 Info.CaptureDeviceAddrMap);
4567 OMPLexicalScope
Scope(CGF, S);
4568 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4576 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4582 const Expr *IfCond =
nullptr;
4584 IfCond = C->getCondition();
4587 const Expr *Device =
nullptr;
4589 Device = C->getDevice();
4595 CGM.getOpenMPRuntime().emitTargetDataCalls(*
this, S, IfCond, Device, RCG,
4603 if (CGM.getLangOpts().OMPTargetTriples.empty())
4607 const Expr *IfCond =
nullptr;
4609 IfCond = C->getCondition();
4612 const Expr *Device =
nullptr;
4614 Device = C->getDevice();
4616 OMPLexicalScope
Scope(*
this, S, OMPD_task);
4617 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
4624 if (CGM.getLangOpts().OMPTargetTriples.empty())
4628 const Expr *IfCond =
nullptr;
4630 IfCond = C->getCondition();
4633 const Expr *Device =
nullptr;
4635 Device = C->getDevice();
4637 OMPLexicalScope
Scope(*
this, S, OMPD_task);
4638 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
4650 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4651 CGF.EmitOMPPrivateClause(S, PrivateScope);
4652 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4656 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4671 llvm::Constant *Addr;
4674 S, ParentName, Fn, Addr,
true, CodeGen);
4675 assert(Fn && Addr &&
"Target device function emission failed.");
4695 CGF, OMPD_target_parallel_for, S.
hasCancel());
4711 llvm::Constant *Addr;
4714 S, ParentName, Fn, Addr,
true, CodeGen);
4715 assert(Fn && Addr &&
"Target device function emission failed.");
4750 llvm::Constant *Addr;
4753 S, ParentName, Fn, Addr,
true, CodeGen);
4754 assert(Fn && Addr &&
"Target device function emission failed.");
4769 const auto *VDecl = cast<VarDecl>(Helper->
getDecl());
4778 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4780 const Expr *IfCond =
nullptr;
4783 C->getNameModifier() == OMPD_taskloop) {
4784 IfCond = C->getCondition();
4798 Data.
Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4802 Data.
Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4816 llvm::BasicBlock *ContBlock =
nullptr;
4817 OMPLoopScope PreInitScope(CGF, S);
4818 if (CGF.ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
4822 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
4823 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
4825 CGF.getProfileCount(&S));
4826 CGF.EmitBlock(ThenBlock);
4827 CGF.incrementProfileCounter(&S);
4831 CGF.EmitOMPSimdInit(S);
4835 enum { LowerBound = 5, UpperBound, Stride, LastIter };
4837 auto *LBP = std::next(I, LowerBound);
4838 auto *UBP = std::next(I, UpperBound);
4839 auto *STP = std::next(I, Stride);
4840 auto *LIP = std::next(I, LastIter);
4848 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4849 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4853 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4854 CGF.EmitVarDecl(*IVDecl);
4855 CGF.EmitIgnoredExpr(S.
getInit());
4861 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4869 CGF.EmitOMPLoopBody(S,
JumpDest());
4870 CGF.EmitStopPoint(&S);
4875 CGF.EmitBranch(ContBlock);
4876 CGF.EmitBlock(ContBlock,
true);
4879 if (HasLastprivateClause) {
4880 CGF.EmitOMPLastprivateClauseFinal(
4882 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4883 CGF.GetAddrOfLocalVar(*LIP),
false,
4887 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4890 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4892 OMPLoopScope PreInitScope(CGF, S);
4893 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.
getLocStart(), S,
4894 OutlinedFn, SharedsTy,
4895 CapturedStruct, IfCond, Data);
4897 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4901 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4903 CGM.getOpenMPRuntime().emitTaskgroupRegion(
4916 EmitOMPTaskLoopBasedDirective(S);
4921 EmitOMPTaskLoopBasedDirective(S);
4929 if (CGM.getLangOpts().OMPTargetTriples.empty())
4933 const Expr *IfCond =
nullptr;
4935 IfCond = C->getCondition();
4938 const Expr *Device =
nullptr;
4940 Device = C->getDevice();
4942 OMPLexicalScope
Scope(*
this, S, OMPD_task);
4943 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
4954 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
4955 for (
const Expr *E : LD->counters()) {
4956 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4957 cast<DeclRefExpr>(E)->getDecl())) {
4959 if (!CGF.LocalDeclMap.count(VD))
4964 if (!C->getNumForLoops())
4966 for (
unsigned I = LD->getCollapsedNumber(),
4967 E = C->getLoopNumIterations().size();
4969 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4970 cast<DeclRefExpr>(C->getLoopCunter(I))->getDecl())) {
4972 if (!CGF.LocalDeclMap.count(VD))
4981 OMPSimdLexicalScope
Scope(*
this, D);
4982 CGM.getOpenMPRuntime().emitInlinedDirective(
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
This represents '#pragma omp distribute simd' composite directive.
Expr * getNextUpperBound() const
This represents '#pragma omp master' directive.
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
Expr * getUpperBoundVariable() const
Other implicit parameter.
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause 'copyin' in the '#pragma omp ...' directives.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Scheduling data for loop-based OpenMP directives.
A (possibly-)qualified type.
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
CodeGenTypes & getTypes()
ArrayRef< OMPClause * > clauses()
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
Expr * getLowerBoundVariable() const
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents '#pragma omp for simd' directive.
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
This represents 'grainsize' clause in the '#pragma omp ...' directive.
This represents '#pragma omp teams distribute parallel for' composite directive.
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents 'if' clause in the '#pragma omp ...' directive.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
This represents 'priority' clause in the '#pragma omp ...' directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
This represents '#pragma omp target teams distribute' combined directive.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Floating point control options.
This represents '#pragma omp parallel for' directive.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents '#pragma omp target teams distribute parallel for' combined directive.
Expr * getCombinedEnsureUpperBound() const
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents '#pragma omp target exit data' directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
This represents clause 'private' in the '#pragma omp ...' directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
const Stmt * getBody() const
Extra information about a function prototype.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Expr * getCalcLastIteration() const
llvm::Value * getPointer() const
unsigned getContextParamPosition() const
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' and...
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents 'nogroup' clause in the '#pragma omp ...' directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents 'safelen' clause in the '#pragma omp ...' directive.
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
Represents a struct/union/class.
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents '#pragma omp parallel' directive.
Address getAddress() const
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
A C++ nested-name-specifier augmented with source location information.
This represents 'simd' clause in the '#pragma omp ...' directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Expr * getEnsureUpperBound() const
SmallVector< const Expr *, 4 > LastprivateVars
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Represents a member of a struct/union/class.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
CharUnits getAlignment() const
llvm::IntegerType * SizeTy
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
bool isReferenceType() const
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr)
This represents '#pragma omp target simd' directive.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
OpenMPDirectiveKind getDirectiveKind() const
This represents '#pragma omp barrier' directive.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
This represents '#pragma omp critical' directive.
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
This represents '#pragma omp distribute parallel for' composite directive.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
This represents '#pragma omp teams distribute parallel for simd' composite directive.
static bool hasScalarEvaluationKind(QualType T)
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
ArrayRef< Expr * > finals()
Expr * getIsLastIterVariable() const
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
OpenMPScheduleClauseKind Schedule
CharUnits - This is an opaque type for sizes expressed in character units.
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Expr * getX()
Get 'x' part of the associated expression/statement.
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
SourceLocation getLocEnd() const LLVM_READONLY
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
CharUnits getAlignment() const
Return the alignment of this pointer.
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Expr * getIterationVariable() const
A builtin binary operation expression such as "x + y" or "x <= y".
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
OpenMPScheduleClauseModifier M2
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
SmallVector< const Expr *, 4 > PrivateCopies
SourceLocation getLocStart() const LLVM_READONLY
Returns starting location of directive kind.
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents '#pragma omp cancellation point' directive.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Expr * getSizeExpr() const
Scope - A scope is a transient data structure that is used while parsing the program.
field_iterator field_begin() const
This represents 'final' clause in the '#pragma omp ...' directive.
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents '#pragma omp teams' directive.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
SmallVector< const Expr *, 4 > FirstprivateCopies
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This represents clause 'reduction' in the '#pragma omp ...' directives.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
This represents '#pragma omp teams distribute simd' combined directive.
void ForceCleanup(std::initializer_list< llvm::Value **> ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CharUnits getPointerAlign() const
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
An ordinary object is located at an address in memory.
SmallVector< const Expr *, 4 > ReductionOps
Controls insertion of cancellation exit blocks in worksharing constructs.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
Represents the body of a CapturedStmt, and serves as its DeclContext.
SmallVector< const Expr *, 4 > ReductionVars
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
This represents '#pragma omp target parallel for simd' directive.
ArrayRef< Expr * > private_counters()
CompoundStmt - This represents a group of statements like { stmt stmt }.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
ASTContext & getContext() const
bool hasCancel() const
Return true if current directive has inner cancel directive.
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents '#pragma omp taskgroup' directive.
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
This represents clause 'aligned' in the '#pragma omp ...' directives.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
Class intended to support codegen of all kind of the reduction clauses.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Expr * getCombinedLowerBoundVariable() const
The class used to assign some variables some temporarily addresses.
This represents clause 'task_reduction' in the '#pragma omp taskgroup' directives.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
This represents '#pragma omp distribute' directive.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
unsigned NumberOfTargetItems
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Expr - This represents one expression.
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Emit only debug info necessary for generating line number tables (-gline-tables-only).
void EmitAutoVarInit(const AutoVarEmission &emission)
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
This represents 'simdlen' clause in the '#pragma omp ...' directive.
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
const T * castAs() const
Member-template castAs<specific type>.
llvm::BasicBlock * getBlock() const
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
bool hasCancel() const
Return true if current directive has inner cancel directive.
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
This represents '#pragma omp target teams distribute parallel for simd' combined directive.
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
llvm::PointerType * getType() const
Return the type of the pointer value.
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
This represents '#pragma omp target teams distribute simd' combined directive.
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
This represents 'ordered' clause in the '#pragma omp ...' directive.
Expr * getDistInc() const
Expr * getNextLowerBound() const
Expr * getPrevEnsureUpperBound() const
This represents '#pragma omp for' directive.
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
SmallVector< const Expr *, 4 > FirstprivateVars
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
This represents '#pragma omp target teams' directive.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
SourceLocation getEnd() const
UnaryOperator - This represents the unary-expression's (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
This represents '#pragma omp cancel' directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
const LangOptions & getLangOpts() const
ASTContext & getContext() const
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
GlobalDecl - represents a global declaration.
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
This represents '#pragma omp flush' directive.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
This represents '#pragma omp parallel for simd' directive.
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
void EmitAlignmentAssumption(llvm::Value *PtrValue, unsigned Alignment, llvm::Value *OffsetValue=nullptr)
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Expr * getLastIteration() const
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
bool isPostfixUpdate() const
Return true if 'v' expression must be updated to original value of 'x', false if 'v' must be updated ...
This represents '#pragma omp target enter data' directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Expr * getStrideVariable() const
This represents 'num_teams' clause in the '#pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
const Stmt * getPreInits() const
This captures a statement into a function.
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
This represents '#pragma omp single' directive.
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents 'hint' clause in the '#pragma omp ...' directive.
void EmitOMPForDirective(const OMPForDirective &S)
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Pseudo declaration for capturing expressions.
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
This is a basic class for representing single OpenMP executable directive.
This represents 'schedule' clause in the '#pragma omp ...' directive.
Expr * getExpr()
Get 'expr' part of the associated expression/statement.
SourceLocation getLocStart() const LLVM_READONLY
OpenMPDirectiveKind
OpenMP directives.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
C-style initialization with assignment.
This represents '#pragma omp taskwait' directive.
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
This file defines OpenMP nodes for declarative directives.
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
This is a basic class for representing single OpenMP clause.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
bool isAnyPointerType() const
virtual void Enter(CodeGenFunction &CGF)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
This represents '#pragma omp target' directive.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Expr * getV()
Get 'v' part of the associated expression/statement.
constexpr XRayInstrMask None
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents '#pragma omp ordered' directive.
This represents '#pragma omp target update' directive.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Dataflow Directional Tag Classes.
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Expr * getPrevUpperBoundVariable() const
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
This represents 'device' clause in the '#pragma omp ...' directive.
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
This represents '#pragma omp section' directive.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
OpenMPScheduleClauseModifier M1
This represents '#pragma omp teams distribute' directive.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const Expr * getInit() const
This represents '#pragma omp simd' directive.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
DeclarationName - The name of a declaration.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
This represents clause 'linear' in the '#pragma omp ...' directives.
SourceLocation getLocStart() const LLVM_READONLY
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or 'OpaqueValueExp...
llvm::Module & getModule() const
This represents '#pragma omp atomic' directive.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Expr * getCombinedInit() const
unsigned getCollapsedNumber() const
Get number of collapsed loops.
Expr * getCombinedNextLowerBound() const
ArrayRef< Expr * > counters()
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
SwitchStmt - This represents a 'switch' stmt.
Expr * getCombinedNextUpperBound() const
void EmitAutoVarCleanups(const AutoVarEmission &emission)
API for captured statement code generation.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
Complex values, per C99 6.2.5p11.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
This file defines OpenMP AST classes for executable directives and clauses.
ArrayRef< Expr * > inits()
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Base for LValueReferenceType and RValueReferenceType.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
StringRef getMangledName(GlobalDecl GD)
Address BasePointersArray
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
SourceLocation getLocStart() const LLVM_READONLY
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Expr * getPrevLowerBoundVariable() const
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
bool isLValueReferenceType() const
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
TranslationUnitDecl * getTranslationUnitDecl() const
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
This represents '#pragma omp target parallel' directive.
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
This represents 'nowait' clause in the '#pragma omp ...' directive.
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
Privates[]
Gets the list of initial values for linear variables.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Expr * getCombinedCond() const
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
This represents '#pragma omp taskloop simd' directive.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Expr * getPreCond() const
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
const VariableArrayType * getAsVariableArrayType(QualType T) const
const llvm::function_ref< void(CodeGenFunction &, llvm::Value *, const OMPTaskDataTy &)> TaskGenTy
This represents '#pragma omp sections' directive.
This represents '#pragma omp target data' directive.
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
A reference to a declared variable, function, enum, etc.
static RValue get(llvm::Value *V)
bool isPointerType() const
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
An l-value expression is a reference to an object with independent storage.
A trivial tuple used to represent a source range.
SourceLocation getLocEnd() const LLVM_READONLY
Returns ending location of directive.
LValue - This represents an lvalue references.
This represents '#pragma omp taskyield' directive.
This represents '#pragma omp distribute parallel for simd' composite directive.
Represents a C array with a specified size that is not an integer-constant-expression.
This represents '#pragma omp parallel sections' directive.
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
SourceLocation getBegin() const
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
llvm::Value * getPointer() const
void EmitOMPTaskDirective(const OMPTaskDirective &S)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
This represents '#pragma omp target parallel for' directive.
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
This represents '#pragma omp taskloop' directive.