21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/Bitcode/BitcodeReader.h"
23 #include "llvm/IR/CallSite.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/GlobalValue.h"
26 #include "llvm/IR/Value.h"
27 #include "llvm/Support/Format.h"
28 #include "llvm/Support/raw_ostream.h"
31 using namespace clang;
32 using namespace CodeGen;
39 enum CGOpenMPRegionKind {
42 ParallelOutlinedRegion,
53 const CGOpenMPRegionKind RegionKind,
56 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
57 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
59 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
62 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
63 Kind(Kind), HasCancel(HasCancel) {}
67 virtual const VarDecl *getThreadIDVariable()
const = 0;
78 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
82 bool hasCancel()
const {
return HasCancel; }
84 static bool classof(
const CGCapturedStmtInfo *Info) {
88 ~CGOpenMPRegionInfo()
override =
default;
91 CGOpenMPRegionKind RegionKind;
98 class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
103 StringRef HelperName)
104 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
106 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
107 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
112 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
115 StringRef getHelperName()
const override {
return HelperName; }
117 static bool classof(
const CGCapturedStmtInfo *Info) {
119 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
120 ParallelOutlinedRegion;
127 StringRef HelperName;
131 class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
137 llvm::SwitchInst *UntiedSwitch =
nullptr;
140 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
142 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
151 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
155 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
157 emitUntiedSwitch(CGF);
172 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
178 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
184 const UntiedTaskActionTy &
Action)
185 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
186 ThreadIDVar(ThreadIDVar), Action(Action) {
187 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
192 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
198 StringRef getHelperName()
const override {
return ".omp_outlined."; }
201 Action.emitUntiedSwitch(CGF);
204 static bool classof(
const CGCapturedStmtInfo *Info) {
206 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
215 const UntiedTaskActionTy &
Action;
220 class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
225 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
227 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
232 return OuterRegionInfo->getContextValue();
233 llvm_unreachable(
"No context value for inlined OpenMP region");
237 if (OuterRegionInfo) {
238 OuterRegionInfo->setContextValue(V);
241 llvm_unreachable(
"No context value for inlined OpenMP region");
247 return OuterRegionInfo->lookup(VD);
253 FieldDecl *getThisFieldDecl()
const override {
255 return OuterRegionInfo->getThisFieldDecl();
261 const VarDecl *getThreadIDVariable()
const override {
263 return OuterRegionInfo->getThreadIDVariable();
268 StringRef getHelperName()
const override {
269 if (
auto *OuterRegionInfo = getOldCSI())
270 return OuterRegionInfo->getHelperName();
271 llvm_unreachable(
"No helper name for inlined OpenMP construct");
276 OuterRegionInfo->emitUntiedSwitch(CGF);
281 static bool classof(
const CGCapturedStmtInfo *Info) {
283 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
286 ~CGOpenMPInlinedRegionInfo()
override =
default;
291 CGOpenMPRegionInfo *OuterRegionInfo;
299 class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
303 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
305 HelperName(HelperName) {}
309 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
312 StringRef getHelperName()
const override {
return HelperName; }
314 static bool classof(
const CGCapturedStmtInfo *Info) {
316 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
320 StringRef HelperName;
324 llvm_unreachable(
"No codegen for expressions");
328 class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
331 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
339 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
342 const VarDecl *VD =
C.getCapturedVar();
350 PrivScope.addPrivate(VD, [&CGF, &DRE]() ->
Address {
354 (void)PrivScope.Privatize();
359 if (
auto *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
366 llvm_unreachable(
"No body for expressions");
371 const VarDecl *getThreadIDVariable()
const override {
372 llvm_unreachable(
"No thread id for expressions");
376 StringRef getHelperName()
const override {
377 llvm_unreachable(
"No helper name for expressions");
380 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
388 class InlinedOpenMPRegionRAII {
390 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
391 FieldDecl *LambdaThisCaptureField =
nullptr;
409 ~InlinedOpenMPRegionRAII() {
425 OMP_IDENT_IMD = 0x01,
427 OMP_IDENT_KMPC = 0x02,
429 OMP_ATOMIC_REDUCE = 0x10,
431 OMP_IDENT_BARRIER_EXPL = 0x20,
433 OMP_IDENT_BARRIER_IMPL = 0x40,
435 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
437 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
439 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140
699 Callback(CodeGen, CGF, *PrePostAction);
702 Callback(CodeGen, CGF, Action);
710 if (
auto *CE = dyn_cast<CallExpr>(ReductionOp))
711 if (
auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
713 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
714 if (
auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
725 std::pair<llvm::Function *, llvm::Function *> Reduction =
727 auto *CE = cast<CallExpr>(InitOp);
728 auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
731 auto *LHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
732 auto *RHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
734 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
735 [=]() ->
Address {
return Private; });
736 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
737 [=]() ->
Address {
return Original; });
738 (void)PrivateScope.Privatize();
744 auto *GV =
new llvm::GlobalVariable(
746 llvm::GlobalValue::PrivateLinkage, Init,
".init");
791 SrcBegin = SrcAddr.getPointer();
794 auto DestEnd = CGF.
Builder.CreateGEP(DestBegin, NumElements);
799 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
800 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
803 auto EntryBB = CGF.
Builder.GetInsertBlock();
808 llvm::PHINode *SrcElementPHI =
nullptr;
811 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
812 "omp.arraycpy.srcElementPast");
813 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
816 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
818 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
819 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
820 DestElementPHI->addIncoming(DestBegin, EntryBB);
830 SrcElementCurrent, ElementTy);
838 auto SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
839 SrcElementPHI, 1,
"omp.arraycpy.dest.element");
840 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
844 auto DestElementNext = CGF.
Builder.CreateConstGEP1_32(
845 DestElementPHI, 1,
"omp.arraycpy.dest.element");
848 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
849 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
850 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
857 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
859 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(E))
861 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
873 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
878 void ReductionCodeGen::emitAggregateInitialization(
885 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
887 DRD ? ClausesData[N].ReductionOp : PrivateVD->getInit(),
894 ClausesData.reserve(Shareds.size());
895 SharedAddresses.reserve(Shareds.size());
896 Sizes.reserve(Shareds.size());
897 BaseDecls.reserve(Shareds.size());
898 auto IPriv = Privates.begin();
899 auto IRed = ReductionOps.begin();
900 for (
const auto *Ref : Shareds) {
901 ClausesData.emplace_back(Ref, *IPriv, *IRed);
902 std::advance(IPriv, 1);
903 std::advance(IRed, 1);
908 assert(SharedAddresses.size() == N &&
909 "Number of generated lvalues must be exactly N.");
910 SharedAddresses.emplace_back(emitSharedLValue(CGF, ClausesData[N].Ref),
911 emitSharedLValueUB(CGF, ClausesData[N].Ref));
916 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
917 QualType PrivateType = PrivateVD->getType();
918 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
919 if (!AsArraySection && !PrivateType->isVariablyModifiedType()) {
922 SharedAddresses[N].first.getType().getNonReferenceType()),
929 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
931 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
932 if (AsArraySection) {
933 Size = CGF.
Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
934 SharedAddresses[N].first.getPointer());
935 Size = CGF.
Builder.CreateNUWAdd(
936 Size, llvm::ConstantInt::get(Size->getType(), 1));
937 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
940 SharedAddresses[N].first.getType().getNonReferenceType());
941 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
943 Sizes.emplace_back(SizeInChars, Size);
946 cast<OpaqueValueExpr>(
955 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
956 QualType PrivateType = PrivateVD->getType();
957 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
958 if (!AsArraySection && !PrivateType->isVariablyModifiedType()) {
959 assert(!Size && !Sizes[N].second &&
960 "Size should be nullptr for non-variably modified redution "
966 cast<OpaqueValueExpr>(
975 assert(SharedAddresses.size() > N &&
"No variable was generated");
977 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
979 QualType PrivateType = PrivateVD->getType();
982 QualType SharedType = SharedAddresses[N].first.getType();
986 SharedType, SharedAddresses[N].first.getBaseInfo());
987 if (isa<OMPArraySectionExpr>(ClausesData[N].Ref) ||
989 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
990 }
else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
994 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
997 PrivateVD->
getType().getQualifiers(),
1004 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1005 QualType PrivateType = PrivateVD->getType();
1013 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1014 QualType PrivateType = PrivateVD->getType();
1019 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
1067 return Address(Addr, BaseLVAlignment);
1073 const VarDecl *OrigVD =
nullptr;
1074 if (
auto *OASE = dyn_cast<OMPArraySectionExpr>(ClausesData[N].Ref)) {
1075 auto *
Base = OASE->getBase()->IgnoreParenImpCasts();
1076 while (
auto *TempOASE = dyn_cast<OMPArraySectionExpr>(
Base))
1077 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1078 while (
auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
1079 Base = TempASE->getBase()->IgnoreParenImpCasts();
1080 DE = cast<DeclRefExpr>(
Base);
1081 OrigVD = cast<VarDecl>(DE->
getDecl());
1082 }
else if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(ClausesData[N].Ref)) {
1083 auto *
Base = ASE->getBase()->IgnoreParenImpCasts();
1084 while (
auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
1085 Base = TempASE->getBase()->IgnoreParenImpCasts();
1086 DE = cast<DeclRefExpr>(
Base);
1087 OrigVD = cast<VarDecl>(DE->
getDecl());
1090 BaseDecls.emplace_back(OrigVD);
1091 auto OriginalBaseLValue = CGF.
EmitLValue(DE);
1094 OriginalBaseLValue);
1096 BaseLValue.
getPointer(), SharedAddresses[N].first.getPointer());
1100 SharedAddresses[N].first.getType(),
1101 OriginalBaseLValue.getPointer()->getType(),
1102 OriginalBaseLValue.getAlignment(), Ptr);
1104 BaseDecls.emplace_back(
1105 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1133 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1136 getThreadIDVariable()->
getType(),
1146 KmpCriticalNameTy = llvm::ArrayType::get(CGM.
Int32Ty, 8);
1152 InternalVars.clear();
1155 static llvm::Function *
1157 const Expr *CombinerInitializer,
const VarDecl *In,
1158 const VarDecl *Out,
bool IsCombiner) {
1161 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1167 Args.push_back(&OmpOutParm);
1168 Args.push_back(&OmpInParm);
1174 IsCombiner ?
".omp_combiner." :
".omp_initializer.", &CGM.
getModule());
1176 Fn->removeFnAttr(llvm::Attribute::NoInline);
1177 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1178 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1203 if (UDRMap.count(D) > 0)
1205 auto &C = CGM.getContext();
1207 In = &C.Idents.get(
"omp_in");
1208 Out = &C.Idents.get(
"omp_out");
1214 llvm::Function *Initializer =
nullptr;
1216 if (!Priv || !Orig) {
1217 Priv = &C.Idents.get(
"omp_priv");
1218 Orig = &C.Idents.get(
"omp_orig");
1225 UDRMap.insert(std::make_pair(D, std::make_pair(Combiner, Initializer)));
1227 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->
CurFn);
1228 Decls.second.push_back(D);
1232 std::pair<llvm::Function *, llvm::Function *>
1234 auto I = UDRMap.find(D);
1235 if (
I != UDRMap.end())
1238 return UDRMap.lookup(D);
1255 const llvm::Twine &
Name =
"") {
1265 "thread id variable must be of type kmp_int32 *");
1267 bool HasCancel =
false;
1268 if (
auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1269 HasCancel = OPD->hasCancel();
1270 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1271 HasCancel = OPSD->hasCancel();
1272 else if (
auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1273 HasCancel = OPFD->hasCancel();
1274 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1275 HasCancel, OutlinedHelperName);
1277 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1285 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1293 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1300 bool Tied,
unsigned &NumberOfParts) {
1303 auto *ThreadID = getThreadID(CGF, D.
getLocStart());
1304 auto *UpLoc = emitUpdateLocation(CGF, D.
getLocStart());
1307 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1312 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy
Action(Tied, PartIDVar,
1316 "thread id variable must be of type kmp_int32 for tasks");
1320 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1322 TD ? TD->hasCancel() :
false,
Action);
1324 auto *Res = CGF.GenerateCapturedStmtFunction(*CS);
1326 NumberOfParts = Action.getNumberOfParts();
1330 Address CGOpenMPRuntime::getOrCreateDefaultLocation(
unsigned Flags) {
1332 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
1334 if (!DefaultOpenMPPSource) {
1339 DefaultOpenMPPSource =
1340 CGM.GetAddrOfConstantCString(
";unknown;unknown;0;0;;").getPointer();
1341 DefaultOpenMPPSource =
1342 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1346 auto fields = builder.beginStruct(IdentTy);
1347 fields.addInt(CGM.Int32Ty, 0);
1348 fields.addInt(CGM.Int32Ty, Flags);
1349 fields.addInt(CGM.Int32Ty, 0);
1350 fields.addInt(CGM.Int32Ty, 0);
1351 fields.add(DefaultOpenMPPSource);
1352 auto DefaultOpenMPLocation =
1353 fields.finishAndCreateGlobal(
"", Align,
true,
1354 llvm::GlobalValue::PrivateLinkage);
1355 DefaultOpenMPLocation->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1357 OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation;
1365 Flags |= OMP_IDENT_KMPC;
1369 return getOrCreateDefaultLocation(Flags).getPointer();
1371 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1373 Address LocValue = Address::invalid();
1374 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
1375 if (
I != OpenMPLocThreadIDMap.end())
1384 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1388 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1397 auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.
getRawEncoding());
1398 if (OMPDebugLoc ==
nullptr) {
1400 llvm::raw_svector_ostream OS2(Buffer2);
1405 dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl)) {
1406 OS2 << FD->getQualifiedNameAsString();
1409 OMPDebugLoc = CGF.
Builder.CreateGlobalStringPtr(OS2.str());
1422 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1427 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
1428 if (
I != OpenMPLocThreadIDMap.end()) {
1429 ThreadID =
I->second.ThreadID;
1430 if (ThreadID !=
nullptr)
1433 if (
auto *OMPRegionInfo =
1435 if (OMPRegionInfo->getThreadIDVariable()) {
1437 auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1442 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1443 Elem.second.ThreadID = ThreadID;
1453 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1457 emitUpdateLocation(CGF, Loc));
1458 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1459 Elem.second.ThreadID = ThreadID;
1464 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1465 if (OpenMPLocThreadIDMap.count(CGF.
CurFn))
1466 OpenMPLocThreadIDMap.erase(CGF.
CurFn);
1467 if (FunctionUDRMap.count(CGF.
CurFn) > 0) {
1468 for(
auto *D : FunctionUDRMap[CGF.
CurFn]) {
1471 FunctionUDRMap.erase(CGF.
CurFn);
1475 llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1478 return llvm::PointerType::getUnqual(IdentTy);
1482 if (!Kmpc_MicroTy) {
1484 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1485 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1486 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams,
true);
1488 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1493 llvm::Constant *RTLFn =
nullptr;
1494 switch (static_cast<OpenMPRTLFunction>(Function)) {
1498 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1500 llvm::FunctionType *FnTy =
1501 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
true);
1502 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_fork_call");
1507 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1508 llvm::FunctionType *FnTy =
1509 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1510 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_global_thread_num");
1516 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1517 CGM.VoidPtrTy, CGM.SizeTy,
1518 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1519 llvm::FunctionType *FnTy =
1520 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams,
false);
1521 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_threadprivate_cached");
1527 llvm::Type *TypeParams[] = {
1528 getIdentTyPointerTy(), CGM.Int32Ty,
1529 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1530 llvm::FunctionType *FnTy =
1531 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1532 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_critical");
1538 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1539 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1541 llvm::FunctionType *FnTy =
1542 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1543 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_critical_with_hint");
1551 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1552 false)->getPointerTo();
1554 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1555 auto KmpcCopyCtorTy =
1556 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1557 false)->getPointerTo();
1560 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
false)
1562 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1563 KmpcCopyCtorTy, KmpcDtorTy};
1564 auto FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1566 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_threadprivate_register");
1572 llvm::Type *TypeParams[] = {
1573 getIdentTyPointerTy(), CGM.Int32Ty,
1574 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1575 llvm::FunctionType *FnTy =
1576 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1577 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_critical");
1583 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1584 llvm::FunctionType *FnTy =
1585 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1586 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_cancel_barrier");
1591 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1592 llvm::FunctionType *FnTy =
1593 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1594 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_barrier");
1599 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1600 llvm::FunctionType *FnTy =
1601 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1602 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_for_static_fini");
1608 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1610 llvm::FunctionType *FnTy =
1611 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1612 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_push_num_threads");
1618 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1619 llvm::FunctionType *FnTy =
1620 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1621 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_serialized_parallel");
1627 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1628 llvm::FunctionType *FnTy =
1629 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1630 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_serialized_parallel");
1635 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1636 llvm::FunctionType *FnTy =
1637 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1638 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_flush");
1643 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1644 llvm::FunctionType *FnTy =
1645 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1646 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_master");
1651 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1652 llvm::FunctionType *FnTy =
1653 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1654 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_master");
1660 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1661 llvm::FunctionType *FnTy =
1662 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1663 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_taskyield");
1668 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1669 llvm::FunctionType *FnTy =
1670 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1671 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_single");
1676 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1677 llvm::FunctionType *FnTy =
1678 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1679 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_single");
1686 assert(KmpRoutineEntryPtrTy !=
nullptr &&
1687 "Type kmp_routine_entry_t must be created.");
1688 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1689 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1691 llvm::FunctionType *FnTy =
1692 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams,
false);
1693 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task_alloc");
1699 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1701 llvm::FunctionType *FnTy =
1702 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1703 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task");
1710 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1712 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams,
false);
1713 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
1714 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
1716 llvm::FunctionType *FnTy =
1717 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1718 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_copyprivate");
1725 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1726 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1728 llvm::Type *TypeParams[] = {
1729 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1730 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1731 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1732 llvm::FunctionType *FnTy =
1733 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1734 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_reduce");
1742 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1743 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1745 llvm::Type *TypeParams[] = {
1746 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1747 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1748 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1749 llvm::FunctionType *FnTy =
1750 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1751 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_reduce_nowait");
1757 llvm::Type *TypeParams[] = {
1758 getIdentTyPointerTy(), CGM.Int32Ty,
1759 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1760 llvm::FunctionType *FnTy =
1761 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1762 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_reduce");
1768 llvm::Type *TypeParams[] = {
1769 getIdentTyPointerTy(), CGM.Int32Ty,
1770 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1771 llvm::FunctionType *FnTy =
1772 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1774 CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_reduce_nowait");
1780 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1782 llvm::FunctionType *FnTy =
1783 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1785 CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task_begin_if0");
1791 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1793 llvm::FunctionType *FnTy =
1794 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1795 RTLFn = CGM.CreateRuntimeFunction(FnTy,
1796 "__kmpc_omp_task_complete_if0");
1801 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1802 llvm::FunctionType *FnTy =
1803 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1804 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_ordered");
1809 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1810 llvm::FunctionType *FnTy =
1811 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1812 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_ordered");
1817 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1818 llvm::FunctionType *FnTy =
1819 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1820 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_taskwait");
1825 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1826 llvm::FunctionType *FnTy =
1827 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1828 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_taskgroup");
1833 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1834 llvm::FunctionType *FnTy =
1835 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1836 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_taskgroup");
1842 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1843 llvm::FunctionType *FnTy =
1844 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1845 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_push_proc_bind");
1852 llvm::Type *TypeParams[] = {
1853 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
1854 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
1855 llvm::FunctionType *FnTy =
1856 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1858 CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task_with_deps");
1865 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1866 CGM.Int32Ty, CGM.VoidPtrTy,
1867 CGM.Int32Ty, CGM.VoidPtrTy};
1868 llvm::FunctionType *FnTy =
1869 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1870 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_wait_deps");
1876 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1877 llvm::FunctionType *FnTy =
1878 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1879 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_cancellationpoint");
1885 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1886 llvm::FunctionType *FnTy =
1887 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1888 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_cancel");
1894 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1896 llvm::FunctionType *FnTy =
1897 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1898 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_push_num_teams");
1904 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1906 llvm::FunctionType *FnTy =
1907 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
true);
1908 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_fork_teams");
1915 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
1919 CGM.Int64Ty->getPointerTo(),
1920 CGM.Int64Ty->getPointerTo(),
1926 llvm::FunctionType *FnTy =
1927 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1928 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_taskloop");
1934 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
1938 llvm::FunctionType *FnTy =
1939 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1940 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_init");
1945 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1946 llvm::FunctionType *FnTy =
1947 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1948 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_fini");
1954 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1955 CGM.Int64Ty->getPointerTo()};
1956 llvm::FunctionType *FnTy =
1957 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1958 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_post");
1964 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1965 CGM.Int64Ty->getPointerTo()};
1966 llvm::FunctionType *FnTy =
1967 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1968 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_wait");
1974 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
1975 llvm::FunctionType *FnTy =
1976 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams,
false);
1978 CGM.CreateRuntimeFunction(FnTy,
"__kmpc_task_reduction_init");
1984 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
1985 llvm::FunctionType *FnTy =
1986 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams,
false);
1987 RTLFn = CGM.CreateRuntimeFunction(
1988 FnTy,
"__kmpc_task_reduction_get_th_data");
1995 llvm::Type *TypeParams[] = {CGM.Int32Ty,
2000 CGM.SizeTy->getPointerTo(),
2001 CGM.Int32Ty->getPointerTo()};
2002 llvm::FunctionType *FnTy =
2003 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
2004 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target");
2011 llvm::Type *TypeParams[] = {CGM.Int32Ty,
2016 CGM.SizeTy->getPointerTo(),
2017 CGM.Int32Ty->getPointerTo(),
2020 llvm::FunctionType *FnTy =
2021 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
2022 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_teams");
2029 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2030 llvm::FunctionType *FnTy =
2031 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
2032 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_register_lib");
2039 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2040 llvm::FunctionType *FnTy =
2041 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
2042 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_unregister_lib");
2048 llvm::Type *TypeParams[] = {CGM.Int32Ty,
2052 CGM.SizeTy->getPointerTo(),
2053 CGM.Int32Ty->getPointerTo()};
2054 llvm::FunctionType *FnTy =
2055 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
2056 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_data_begin");
2062 llvm::Type *TypeParams[] = {CGM.Int32Ty,
2066 CGM.SizeTy->getPointerTo(),
2067 CGM.Int32Ty->getPointerTo()};
2068 llvm::FunctionType *FnTy =
2069 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
2070 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_data_end");
2076 llvm::Type *TypeParams[] = {CGM.Int32Ty,
2080 CGM.SizeTy->getPointerTo(),
2081 CGM.Int32Ty->getPointerTo()};
2082 llvm::FunctionType *FnTy =
2083 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
2084 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_data_update");
2088 assert(RTLFn &&
"Unable to find OpenMP runtime function");
2094 assert((IVSize == 32 || IVSize == 64) &&
2095 "IV size is not compatible with the omp runtime");
2096 auto Name = IVSize == 32 ? (IVSigned ?
"__kmpc_for_static_init_4"
2097 :
"__kmpc_for_static_init_4u")
2098 : (IVSigned ?
"__kmpc_for_static_init_8"
2099 :
"__kmpc_for_static_init_8u");
2100 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2101 auto PtrTy = llvm::PointerType::getUnqual(ITy);
2102 llvm::Type *TypeParams[] = {
2103 getIdentTyPointerTy(),
2106 llvm::PointerType::getUnqual(CGM.Int32Ty),
2113 llvm::FunctionType *FnTy =
2114 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
2115 return CGM.CreateRuntimeFunction(FnTy,
Name);
2120 assert((IVSize == 32 || IVSize == 64) &&
2121 "IV size is not compatible with the omp runtime");
2124 ? (IVSigned ?
"__kmpc_dispatch_init_4" :
"__kmpc_dispatch_init_4u")
2125 : (IVSigned ?
"__kmpc_dispatch_init_8" :
"__kmpc_dispatch_init_8u");
2126 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2127 llvm::Type *TypeParams[] = { getIdentTyPointerTy(),
2135 llvm::FunctionType *FnTy =
2136 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
2137 return CGM.CreateRuntimeFunction(FnTy,
Name);
2142 assert((IVSize == 32 || IVSize == 64) &&
2143 "IV size is not compatible with the omp runtime");
2146 ? (IVSigned ?
"__kmpc_dispatch_fini_4" :
"__kmpc_dispatch_fini_4u")
2147 : (IVSigned ?
"__kmpc_dispatch_fini_8" :
"__kmpc_dispatch_fini_8u");
2148 llvm::Type *TypeParams[] = {
2149 getIdentTyPointerTy(),
2152 llvm::FunctionType *FnTy =
2153 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
2154 return CGM.CreateRuntimeFunction(FnTy,
Name);
2159 assert((IVSize == 32 || IVSize == 64) &&
2160 "IV size is not compatible with the omp runtime");
2163 ? (IVSigned ?
"__kmpc_dispatch_next_4" :
"__kmpc_dispatch_next_4u")
2164 : (IVSigned ?
"__kmpc_dispatch_next_8" :
"__kmpc_dispatch_next_8u");
2165 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2166 auto PtrTy = llvm::PointerType::getUnqual(ITy);
2167 llvm::Type *TypeParams[] = {
2168 getIdentTyPointerTy(),
2170 llvm::PointerType::getUnqual(CGM.Int32Ty),
2175 llvm::FunctionType *FnTy =
2176 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
2177 return CGM.CreateRuntimeFunction(FnTy,
Name);
2182 assert(!CGM.getLangOpts().OpenMPUseTLS ||
2183 !CGM.getContext().getTargetInfo().isTLSSupported());
2186 Twine(CGM.getMangledName(VD)) +
".cache.");
2193 if (CGM.getLangOpts().OpenMPUseTLS &&
2194 CGM.getContext().getTargetInfo().isTLSSupported())
2198 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2201 CGM.
getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2213 auto OMPLoc = emitUpdateLocation(CGF, Loc);
2221 Ctor, CopyCtor, Dtor};
2229 if (CGM.getLangOpts().OpenMPUseTLS &&
2230 CGM.getContext().getTargetInfo().isTLSSupported())
2238 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
2240 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2247 Args.push_back(&Dst);
2249 auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2250 CGM.getContext().VoidPtrTy, Args);
2251 auto FTy = CGM.getTypes().GetFunctionType(FI);
2252 auto Fn = CGM.CreateGlobalInitOrDestructFunction(
2253 FTy,
".__kmpc_global_ctor_.", FI, Loc);
2258 CGM.getContext().VoidPtrTy, Dst.getLocation());
2266 CGM.getContext().VoidPtrTy, Dst.getLocation());
2278 Args.push_back(&Dst);
2280 auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2281 CGM.getContext().VoidTy, Args);
2282 auto FTy = CGM.getTypes().GetFunctionType(FI);
2283 auto Fn = CGM.CreateGlobalInitOrDestructFunction(
2284 FTy,
".__kmpc_global_dtor_.", FI, Loc);
2289 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2292 false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2303 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2305 llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2306 false)->getPointerTo();
2310 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2311 if (Ctor ==
nullptr) {
2312 auto CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2313 false)->getPointerTo();
2314 Ctor = llvm::Constant::getNullValue(CtorTy);
2316 if (Dtor ==
nullptr) {
2317 auto DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2318 false)->getPointerTo();
2319 Dtor = llvm::Constant::getNullValue(DtorTy);
2322 auto InitFunctionTy =
2323 llvm::FunctionType::get(CGM.VoidTy,
false);
2324 auto InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2325 InitFunctionTy,
".__omp_threadprivate_init_.",
2326 CGM.getTypes().arrangeNullaryFunction());
2330 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2334 return InitFunction;
2344 llvm::Twine VarName(Name,
".artificial.");
2358 VarLVType->getPointerTo(0)),
2359 CGM.getPointerAlign());
2411 const Expr *IfCond) {
2414 auto *RTLoc = emitUpdateLocation(CGF, Loc);
2415 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](
CodeGenFunction &CGF,
2418 auto &RT = CGF.CGM.getOpenMPRuntime();
2421 CGF.Builder.getInt32(CapturedVars.size()),
2422 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2424 RealArgs.append(std::begin(Args), std::end(Args));
2425 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2428 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2430 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](
CodeGenFunction &CGF,
2432 auto &RT = CGF.CGM.getOpenMPRuntime();
2433 auto ThreadID = RT.getThreadID(CGF, Loc);
2437 CGF.EmitRuntimeCall(
2441 auto ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2445 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32( 0));
2447 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
2448 OutlinedFnArgs.push_back(ZeroAddr.
getPointer());
2449 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2450 CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
2453 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2454 CGF.EmitRuntimeCall(
2459 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
2474 if (
auto *OMPRegionInfo =
2476 if (OMPRegionInfo->getThreadIDVariable())
2477 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2479 auto ThreadID = getThreadID(CGF, Loc);
2482 auto ThreadIDTemp = CGF.
CreateMemTemp(Int32Ty,
".threadid_temp.");
2486 return ThreadIDTemp;
2491 const llvm::Twine &
Name) {
2493 llvm::raw_svector_ostream Out(Buffer);
2495 auto RuntimeName = Out.str();
2496 auto &Elem = *InternalVars.insert(std::make_pair(RuntimeName,
nullptr)).first;
2498 assert(Elem.second->getType()->getPointerElementType() == Ty &&
2499 "OMP internal variable has different type than requested");
2500 return &*Elem.second;
2503 return Elem.second =
new llvm::GlobalVariable(
2504 CGM.getModule(), Ty,
false,
2505 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2510 llvm::Twine
Name(
".gomp_critical_user_", CriticalName);
2522 llvm::BasicBlock *ContBlock =
nullptr;
2528 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2537 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2553 StringRef CriticalName,
2562 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2567 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2588 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2603 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2604 llvm::ConstantInt::get(CGM.IntTy, 0,
true)};
2606 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2607 Region->emitUntiedSwitch(CGF);
2619 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2630 unsigned Index,
const VarDecl *Var) {
2644 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2645 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) {
2651 Args.push_back(&LHSArg);
2652 Args.push_back(&RHSArg);
2656 ".omp.copyprivate.copy_func", &CGM.
getModule());
2672 for (
unsigned I = 0, E = AssignmentOps.size();
I <
E; ++
I) {
2673 auto DestVar = cast<VarDecl>(cast<DeclRefExpr>(DestExprs[
I])->getDecl());
2676 auto SrcVar = cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[
I])->getDecl());
2679 auto *VD = cast<DeclRefExpr>(CopyprivateVars[
I])->getDecl();
2681 CGF.
EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[
I]);
2690 ArrayRef<const Expr *> CopyprivateVars,
2691 ArrayRef<const Expr *> SrcExprs,
2692 ArrayRef<const Expr *> DstExprs,
2693 ArrayRef<const Expr *> AssignmentOps) {
2696 assert(CopyprivateVars.size() == SrcExprs.size() &&
2697 CopyprivateVars.size() == DstExprs.size() &&
2698 CopyprivateVars.size() == AssignmentOps.size());
2699 auto &
C = CGM.getContext();
2709 Address DidIt = Address::invalid();
2710 if (!CopyprivateVars.empty()) {
2712 auto KmpInt32Ty = C.getIntTypeForBitwidth(32, 1);
2713 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2717 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2731 llvm::APInt ArraySize(32, CopyprivateVars.size());
2732 auto CopyprivateArrayTy =
2737 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2738 for (
unsigned I = 0, E = CopyprivateVars.size();
I <
E; ++
I) {
2750 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
2751 auto *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2757 emitUpdateLocation(CGF, Loc),
2758 getThreadID(CGF, Loc),
2778 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2791 bool ForceSimpleCall) {
2797 if (Kind == OMPD_for)
2798 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2799 else if (Kind == OMPD_sections)
2800 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2801 else if (Kind == OMPD_single)
2802 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2803 else if (Kind == OMPD_barrier)
2804 Flags = OMP_IDENT_BARRIER_EXPL;
2806 Flags = OMP_IDENT_BARRIER_IMPL;
2809 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2810 getThreadID(CGF, Loc)};
2811 if (
auto *OMPRegionInfo =
2813 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2823 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2826 auto CancelDestination =
2839 bool Chunked,
bool Ordered) {
2840 switch (ScheduleKind) {
2841 case OMPC_SCHEDULE_static:
2844 case OMPC_SCHEDULE_dynamic:
2846 case OMPC_SCHEDULE_guided:
2848 case OMPC_SCHEDULE_runtime:
2850 case OMPC_SCHEDULE_auto:
2853 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2856 llvm_unreachable(
"Unexpected runtime schedule");
2867 bool Chunked)
const {
2891 case OMPC_SCHEDULE_MODIFIER_monotonic:
2894 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2897 case OMPC_SCHEDULE_MODIFIER_simd:
2906 case OMPC_SCHEDULE_MODIFIER_monotonic:
2909 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2912 case OMPC_SCHEDULE_MODIFIER_simd:
2930 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2942 : CGF.
Builder.getIntN(IVSize, 1);
2944 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2946 Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2949 CGF.
Builder.getIntN(IVSize, 1),
2976 if (Chunk ==
nullptr) {
2979 "expected static non-chunked schedule");
2981 Chunk = CGF.
Builder.getIntN(IVSize, 1);
2987 "expected static chunked schedule");
2996 CGF.
Builder.getIntN(IVSize, 1),
3005 unsigned IVSize,
bool IVSigned,
3011 auto *UpdatedLocation = emitUpdateLocation(CGF, Loc);
3012 auto *ThreadId = getThreadID(CGF, Loc);
3015 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, IVSize,
3016 Ordered, IL, LB, UB, ST, Chunk);
3025 auto *UpdatedLocation = emitUpdateLocation(CGF, Loc);
3026 auto *ThreadId = getThreadID(CGF, Loc);
3039 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3051 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3065 emitUpdateLocation(CGF, Loc),
3066 getThreadID(CGF, Loc),
3086 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3108 case OMPC_PROC_BIND_master:
3109 RuntimeProcBind = ProcBindMaster;
3111 case OMPC_PROC_BIND_close:
3112 RuntimeProcBind = ProcBindClose;
3114 case OMPC_PROC_BIND_spread:
3115 RuntimeProcBind = ProcBindSpread;
3118 llvm_unreachable(
"Unsupported proc_bind value.");
3122 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3123 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind,
true)};
3133 emitUpdateLocation(CGF, Loc));
3162 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty()
const {
3164 return OffloadEntriesTargetRegion.empty();
3168 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3169 initializeTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
3170 StringRef ParentName,
unsigned LineNum,
3172 assert(CGM.getLangOpts().OpenMPIsDevice &&
"Initialization of entries is "
3173 "only required for the device "
3174 "code generation.");
3175 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3176 OffloadEntryInfoTargetRegion(Order,
nullptr,
nullptr,
3178 ++OffloadingEntriesNum;
3181 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3182 registerTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
3183 StringRef ParentName,
unsigned LineNum,
3184 llvm::Constant *Addr, llvm::Constant *
ID,
3188 if (CGM.getLangOpts().OpenMPIsDevice) {
3189 assert(hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) &&
3190 "Entry must exist.");
3192 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3193 assert(Entry.isValid() &&
"Entry not initialized!");
3194 Entry.setAddress(Addr);
3196 Entry.setFlags(Flags);
3199 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID, Flags);
3200 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3204 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3205 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
3206 unsigned LineNum)
const {
3207 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3208 if (PerDevice == OffloadEntriesTargetRegion.end())
3210 auto PerFile = PerDevice->second.find(FileID);
3211 if (PerFile == PerDevice->second.end())
3213 auto PerParentName = PerFile->second.find(ParentName);
3214 if (PerParentName == PerFile->second.end())
3216 auto PerLine = PerParentName->second.find(LineNum);
3217 if (PerLine == PerParentName->second.end())
3220 if (PerLine->second.getAddress() || PerLine->second.getID())
3225 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3226 const OffloadTargetRegionEntryInfoActTy &
Action) {
3228 for (
auto &D : OffloadEntriesTargetRegion)
3229 for (
auto &F : D.second)
3230 for (
auto &
P : F.second)
3231 for (
auto &L :
P.second)
3232 Action(D.first, F.first,
P.first(), L.first, L.second);
3238 static llvm::Function *
3244 Args.push_back(&DummyPtr);
3265 auto &M = CGM.getModule();
3266 auto &C = CGM.getContext();
3269 auto &Devices = CGM.getLangOpts().OMPTargetTriples;
3273 assert(!Devices.empty() &&
"No OpenMP offloading devices??");
3277 auto *OffloadEntryTy =
3279 llvm::GlobalVariable *HostEntriesBegin =
new llvm::GlobalVariable(
3280 M, OffloadEntryTy,
true,
3282 ".omp_offloading.entries_begin");
3283 llvm::GlobalVariable *HostEntriesEnd =
new llvm::GlobalVariable(
3284 M, OffloadEntryTy,
true,
3286 ".omp_offloading.entries_end");
3289 auto *DeviceImageTy = cast<llvm::StructType>(
3292 auto DeviceImagesEntries = DeviceImagesBuilder.beginArray(DeviceImageTy);
3294 for (
unsigned i = 0; i < Devices.size(); ++i) {
3295 StringRef T = Devices[i].getTriple();
3296 auto *ImgBegin =
new llvm::GlobalVariable(
3299 Twine(
".omp_offloading.img_start.") + Twine(T));
3300 auto *ImgEnd =
new llvm::GlobalVariable(
3302 nullptr, Twine(
".omp_offloading.img_end.") + Twine(T));
3304 auto Dev = DeviceImagesEntries.beginStruct(DeviceImageTy);
3307 Dev.add(HostEntriesBegin);
3308 Dev.add(HostEntriesEnd);
3309 Dev.finishAndAddTo(DeviceImagesEntries);
3313 llvm::GlobalVariable *DeviceImages =
3314 DeviceImagesEntries.finishAndCreateGlobal(
".omp_offloading.device_images",
3315 CGM.getPointerAlign(),
3317 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3320 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
3321 llvm::Constant::getNullValue(CGM.Int32Ty)};
3324 auto *BinaryDescriptorTy = cast<llvm::StructType>(
3327 auto DescInit = DescBuilder.beginStruct(BinaryDescriptorTy);
3328 DescInit.addInt(CGM.Int32Ty, Devices.size());
3329 DescInit.add(llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
3332 DescInit.add(HostEntriesBegin);
3333 DescInit.add(HostEntriesEnd);
3335 auto *Desc = DescInit.finishAndCreateGlobal(
".omp_offloading.descriptor",
3336 CGM.getPointerAlign(),
3344 auto *IdentInfo = &C.Idents.get(
".omp_offloading.reg_unreg_var");
3349 CGM,
".omp_offloading.descriptor_unreg",
3355 CGM,
".omp_offloading.descriptor_reg",
3359 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
3361 if (CGM.supportsCOMDAT()) {
3366 auto ComdatKey = M.getOrInsertComdat(RegFn->getName());
3367 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
3369 RegFn->setComdat(ComdatKey);
3370 UnRegFn->setComdat(ComdatKey);
3371 DeviceImages->setComdat(ComdatKey);
3372 Desc->setComdat(ComdatKey);
3377 void CGOpenMPRuntime::createOffloadEntry(llvm::Constant *ID,
3378 llvm::Constant *Addr, uint64_t Size,
3380 StringRef
Name = Addr->getName();
3381 auto *TgtOffloadEntryType = cast<llvm::StructType>(
3383 llvm::LLVMContext &C = CGM.getModule().getContext();
3384 llvm::Module &M = CGM.getModule();
3387 llvm::Constant *AddrPtr = llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy);
3390 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3392 llvm::GlobalVariable *Str =
3393 new llvm::GlobalVariable(M, StrPtrInit->getType(),
true,
3395 ".omp_offloading.entry_name");
3396 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3397 llvm::Constant *StrPtr = llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy);
3405 auto EntryInit = EntryBuilder.
beginStruct(TgtOffloadEntryType);
3406 EntryInit.add(AddrPtr);
3407 EntryInit.add(StrPtr);
3408 EntryInit.addInt(CGM.SizeTy, Size);
3409 EntryInit.addInt(CGM.Int32Ty, Flags);
3410 EntryInit.addInt(CGM.Int32Ty, 0);
3411 llvm::GlobalVariable *Entry =
3412 EntryInit.finishAndCreateGlobal(
".omp_offloading.entry",
3418 Entry->setSection(
".omp_offloading.entries");
3435 llvm::Module &M = CGM.getModule();
3436 llvm::LLVMContext &C = M.getContext();
3441 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(
"omp_offload.info");
3444 auto getMDInt = [&](
unsigned v) {
3445 return llvm::ConstantAsMetadata::get(
3446 llvm::ConstantInt::get(llvm::Type::getInt32Ty(C),
v));
3449 auto getMDString = [&](StringRef
v) {
return llvm::MDString::get(C,
v); };
3452 auto &&TargetRegionMetadataEmitter = [&](
3453 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
unsigned Line,
3465 Ops.push_back(getMDInt(E.getKind()));
3466 Ops.push_back(getMDInt(DeviceID));
3467 Ops.push_back(getMDInt(FileID));
3468 Ops.push_back(getMDString(ParentName));
3469 Ops.push_back(getMDInt(Line));
3470 Ops.push_back(getMDInt(E.getOrder()));
3473 OrderedEntries[E.getOrder()] = &
E;
3476 MD->addOperand(llvm::MDNode::get(C, Ops));
3480 TargetRegionMetadataEmitter);
3482 for (
auto *E : OrderedEntries) {
3483 assert(E &&
"All ordered entries must exist!");
3485 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3487 assert(CE->getID() && CE->getAddress() &&
3488 "Entry ID and Addr are invalid!");
3489 createOffloadEntry(CE->getID(), CE->getAddress(), 0);
3491 llvm_unreachable(
"Unsupported entry kind.");
3501 if (!CGM.getLangOpts().OpenMPIsDevice)
3504 if (CGM.getLangOpts().OMPHostIRFile.empty())
3507 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
3511 llvm::LLVMContext C;
3512 auto ME = expectedToErrorOrAndEmitErrors(
3513 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
3518 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata(
"omp_offload.info");
3522 for (
auto I : MD->operands()) {
3523 llvm::MDNode *MN = cast<llvm::MDNode>(
I);
3525 auto getMDInt = [&](
unsigned Idx) {
3526 llvm::ConstantAsMetadata *V =
3527 cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3528 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
3531 auto getMDString = [&](
unsigned Idx) {
3532 llvm::MDString *V = cast<llvm::MDString>(MN->getOperand(Idx));
3533 return V->getString();
3536 switch (getMDInt(0)) {
3538 llvm_unreachable(
"Unexpected metadata!");
3540 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3541 OFFLOAD_ENTRY_INFO_TARGET_REGION:
3543 getMDInt(1), getMDInt(2),
3544 getMDString(3), getMDInt(4),
3552 if (!KmpRoutineEntryPtrTy) {
3554 auto &C = CGM.getContext();
3555 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3557 KmpRoutineEntryPtrQTy = C.getPointerType(
3558 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3559 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3597 RD->completeDefinition();
3623 RD->completeDefinition();
3648 RD->completeDefinition();
3655 struct PrivateHelpersTy {
3656 PrivateHelpersTy(
const VarDecl *Original,
const VarDecl *PrivateCopy,
3657 const VarDecl *PrivateElemInit)
3658 : Original(Original), PrivateCopy(PrivateCopy),
3659 PrivateElemInit(PrivateElemInit) {}
3662 const VarDecl *PrivateElemInit;
3664 typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
3669 if (!Privates.empty()) {
3676 for (
auto &&Pair : Privates) {
3677 auto *VD = Pair.second.Original;
3688 RD->completeDefinition();
3697 QualType KmpRoutineEntryPointerQTy) {
3716 UD->completeDefinition();
3717 QualType KmpCmplrdataTy = C.getRecordType(UD);
3718 auto *RD = C.buildImplicitRecord(
"kmp_task_t");
3719 RD->startDefinition();
3736 RD->completeDefinition();
3742 ArrayRef<PrivateDataTy>
Privates) {
3754 RD->completeDefinition();
3772 QualType KmpTaskTWithPrivatesPtrQTy,
3783 Args.push_back(&GtidArg);
3784 Args.push_back(&TaskTypeArg);
3785 auto &TaskEntryFnInfo =
3806 auto *KmpTaskTWithPrivatesQTyRD =
3807 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3810 auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3811 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3815 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3821 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3823 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3826 PrivatesLVal.getPointer(), CGF.
VoidPtrTy);
3828 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3830 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
3837 std::end(CommonArgs));
3839 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3842 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3845 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3848 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3851 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3854 CallArgs.push_back(LBParam);
3855 CallArgs.push_back(UBParam);
3856 CallArgs.push_back(StParam);
3857 CallArgs.push_back(LIParam);
3858 CallArgs.push_back(RParam);
3860 CallArgs.push_back(SharedsParam);
3864 RValue::get(CGF.
Builder.getInt32(0)),
3873 QualType KmpTaskTWithPrivatesPtrQTy,
3874 QualType KmpTaskTWithPrivatesQTy) {
3882 Args.push_back(&GtidArg);
3883 Args.push_back(&TaskTypeArg);
3885 auto &DestructorFnInfo =
3888 auto *DestructorFn =
3890 ".omp_task_destructor.", &CGM.
getModule());
3901 auto *KmpTaskTWithPrivatesQTyRD =
3902 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3903 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3906 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3907 if (
auto DtorKind = Field->getType().isDestructedType()) {
3909 CGF.
pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3913 return DestructorFn;
3928 ArrayRef<const Expr *> PrivateVars,
3929 ArrayRef<const Expr *> FirstprivateVars,
3930 ArrayRef<const Expr *> LastprivateVars,
3932 ArrayRef<PrivateDataTy>
Privates) {
3936 C,
nullptr, Loc,
nullptr,
3937 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3939 Args.push_back(&TaskPrivatesArg);
3940 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
3941 unsigned Counter = 1;
3942 for (
auto *E: PrivateVars) {
3944 C,
nullptr, Loc,
nullptr,
3945 C.getPointerType(C.getPointerType(E->getType()))
3949 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3950 PrivateVarsPos[VD] = Counter;
3953 for (
auto *E : FirstprivateVars) {
3955 C,
nullptr, Loc,
nullptr,
3956 C.getPointerType(C.getPointerType(E->getType()))
3960 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3961 PrivateVarsPos[VD] = Counter;
3964 for (
auto *E: LastprivateVars) {
3966 C,
nullptr, Loc,
nullptr,
3967 C.getPointerType(C.getPointerType(E->getType()))
3971 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3972 PrivateVarsPos[VD] = Counter;
3975 auto &TaskPrivatesMapFnInfo =
3977 auto *TaskPrivatesMapTy =
3981 ".omp_task_privates_map.", &CGM.
getModule());
3983 TaskPrivatesMapFnInfo);
3984 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3985 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3986 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3990 TaskPrivatesMapFnInfo, Args);
3996 auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3998 for (
auto *Field : PrivatesQTyRD->fields()) {
4000 auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4003 RefLVal.getAddress(), RefLVal.getType()->castAs<
PointerType>());
4008 return TaskPrivatesMap;
4012 const PrivateDataTy *P2) {
4013 return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0);
4023 ArrayRef<PrivateDataTy>
Privates,
bool ForDup) {
4025 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
4036 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4037 for (
auto &&Pair : Privates) {
4038 auto *VD = Pair.second.PrivateCopy;
4040 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4043 if (
auto *Elem = Pair.second.PrivateElemInit) {
4044 auto *OriginalVD = Pair.second.Original;
4045 auto *SharedField = CapturesInfo.
lookup(OriginalVD);
4048 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4051 SharedRefLValue.getBaseInfo().getMayAlias()));
4052 QualType Type = OriginalVD->getType();
4058 SharedRefLValue.getAddress(), Type);
4063 PrivateLValue.
getAddress(), SharedRefLValue.getAddress(), Type,
4064 [&CGF, Elem, Init, &CapturesInfo](
Address DestElement,
4069 Elem, [SrcElement]() ->
Address {
return SrcElement; });
4073 CGF, &CapturesInfo);
4074 CGF.EmitAnyExprToMem(Init, DestElement,
4075 Init->getType().getQualifiers(),
4082 return SharedRefLValue.getAddress();
4098 ArrayRef<PrivateDataTy>
Privates) {
4099 bool InitRequired =
false;
4100 for (
auto &&Pair : Privates) {
4101 auto *VD = Pair.second.PrivateCopy;
4103 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4106 return InitRequired;
4123 QualType KmpTaskTWithPrivatesPtrQTy,
4127 ArrayRef<PrivateDataTy>
Privates,
bool WithLastIter) {
4131 KmpTaskTWithPrivatesPtrQTy,
4134 KmpTaskTWithPrivatesPtrQTy,
4138 Args.push_back(&DstArg);
4139 Args.push_back(&SrcArg);
4140 Args.push_back(&LastprivArg);
4141 auto &TaskDupFnInfo =
4157 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
4159 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
4167 assert(!Privates.empty());
4168 Address KmpTaskSharedsPtr = Address::invalid();
4174 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
4182 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4183 SharedsTy, SharedsPtrTy, Data, Privates,
true);
4192 bool NeedsCleanup =
false;
4193 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
4194 auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4195 for (
auto *FD : PrivateRD->fields()) {
4196 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4200 return NeedsCleanup;
4203 CGOpenMPRuntime::TaskResultTy
4208 auto &C = CGM.getContext();
4213 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
4214 Privates.push_back(std::make_pair(
4216 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4223 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
4224 Privates.push_back(std::make_pair(
4227 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4228 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))));
4234 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
4235 Privates.push_back(std::make_pair(
4237 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4241 llvm::array_pod_sort(Privates.begin(), Privates.end(),
4243 auto KmpInt32Ty = C.getIntTypeForBitwidth(32, 1);
4253 auto *KmpTaskTWithPrivatesQTyRD =
4255 auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4256 QualType KmpTaskTWithPrivatesPtrQTy =
4257 C.getPointerType(KmpTaskTWithPrivatesQTy);
4258 auto *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
4259 auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo();
4260 auto *KmpTaskTWithPrivatesTySize = CGF.
getTypeSize(KmpTaskTWithPrivatesQTy);
4261 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4265 auto *TaskPrivatesMapTy =
4266 std::next(cast<llvm::Function>(TaskFunction)->arg_begin(), 3)->getType();
4267 if (!Privates.empty()) {
4268 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4273 TaskPrivatesMap, TaskPrivatesMapTy);
4275 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4276 cast<llvm::PointerType>(TaskPrivatesMapTy));
4282 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4294 DestructorsFlag = 0x8,
4297 unsigned Flags = Data.
Tied ? TiedFlag : 0;
4298 bool NeedsCleanup =
false;
4299 if (!Privates.empty()) {
4302 Flags = Flags | DestructorsFlag;
4305 Flags = Flags | PriorityFlag;
4307 Data.
Final.getPointer()
4309 CGF.
Builder.getInt32(FinalFlag),
4311 : CGF.
Builder.getInt32(Data.
Final.getInt() ? FinalFlag : 0);
4312 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
4313 auto *SharedsSize = CGM.
getSize(C.getTypeSizeInChars(SharedsTy));
4314 llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
4315 getThreadID(CGF, Loc), TaskFlags,
4316 KmpTaskTWithPrivatesTySize, SharedsSize,
4318 TaskEntry, KmpRoutineEntryPtrTy)};
4322 NewTask, KmpTaskTWithPrivatesPtrTy);
4324 KmpTaskTWithPrivatesQTy);
4329 Address KmpTaskSharedsPtr = Address::invalid();
4334 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4342 if (!Privates.empty()) {
4343 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4344 SharedsTy, SharedsPtrTy, Data, Privates,
4349 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4350 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4355 enum { Priority = 0, Destructors = 1 };
4357 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4358 auto *KmpCmplrdataUD = (*FI)->getType()->getAsUnionType()->getDecl();
4361 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4362 KmpTaskTWithPrivatesQTy);
4365 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4367 DestructorFn, KmpRoutineEntryPtrTy),
4373 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4375 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4396 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4402 auto &C = CGM.getContext();
4404 Address DependenciesArray = Address::invalid();
4405 unsigned NumDependencies = Data.
Dependences.size();
4406 if (NumDependencies) {
4408 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 };
4409 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4412 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy),
false);
4415 KmpDependInfoRD = C.buildImplicitRecord(
"kmp_depend_info");
4426 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4432 for (
unsigned i = 0; i < NumDependencies; ++i) {
4443 CGF.
Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
4445 Size = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
4462 RTLDependenceKindTy DepKind;
4464 case OMPC_DEPEND_in:
4468 case OMPC_DEPEND_out:
4469 case OMPC_DEPEND_inout:
4472 case OMPC_DEPEND_source:
4473 case OMPC_DEPEND_sink:
4475 llvm_unreachable(
"Unknown task dependence type");
4493 auto *ThreadID = getThreadID(CGF, Loc);
4494 auto *UpLoc = emitUpdateLocation(CGF, Loc);
4495 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4497 if (NumDependencies) {
4498 DepTaskArgs[0] = UpLoc;
4499 DepTaskArgs[1] = ThreadID;
4500 DepTaskArgs[2] = NewTask;
4501 DepTaskArgs[3] = CGF.
Builder.getInt32(NumDependencies);
4502 DepTaskArgs[4] = DependenciesArray.
getPointer();
4503 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4504 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4506 auto &&ThenCodeGen = [
this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
4510 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4511 auto PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
4512 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
4514 if (NumDependencies) {
4515 CGF.EmitRuntimeCall(
4523 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4524 Region->emitUntiedSwitch(CGF);
4528 if (NumDependencies) {
4529 DepWaitTaskArgs[0] = UpLoc;
4530 DepWaitTaskArgs[1] = ThreadID;
4531 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
4532 DepWaitTaskArgs[3] = DependenciesArray.
getPointer();
4533 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
4534 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4536 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
4539 auto &RT = CGF.CGM.getOpenMPRuntime();
4545 if (NumDependencies)
4549 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy](
4552 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4553 CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
4569 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
4585 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4592 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4598 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4631 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4640 llvm::ConstantInt::getNullValue(
4642 llvm::ConstantInt::getSigned(
4644 ? Data.
Schedule.getInt() ? NumTasks : Grainsize
4649 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
4652 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
4669 const Expr *,
const Expr *)> &RedOpGen,
4670 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4671 const Expr *UpExpr =
nullptr) {
4684 auto LHSEnd = CGF.
Builder.CreateGEP(LHSBegin, NumElements);
4689 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4690 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4693 auto EntryBB = CGF.
Builder.GetInsertBlock();
4698 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4699 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4700 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4705 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4706 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4707 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4714 Scope.addPrivate(LHSVar, [=]() ->
Address {
return LHSElementCurrent; });
4715 Scope.addPrivate(RHSVar, [=]() ->
Address {
return RHSElementCurrent; });
4717 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4718 Scope.ForceCleanup();
4721 auto LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4722 LHSElementPHI, 1,
"omp.arraycpy.dest.element");
4723 auto RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4724 RHSElementPHI, 1,
"omp.arraycpy.src.element");
4727 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4728 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4729 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4730 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4740 const Expr *ReductionOp) {
4741 if (
auto *CE = dyn_cast<CallExpr>(ReductionOp))
4742 if (
auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4744 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4745 if (
auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4746 std::pair<llvm::Function *, llvm::Function *> Reduction =
4748 RValue Func = RValue::get(Reduction.first);
4758 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
4759 ArrayRef<const Expr *> ReductionOps) {
4766 Args.push_back(&LHSArg);
4767 Args.push_back(&RHSArg);
4771 ".omp.reduction.reduction_func", &CGM.
getModule());
4789 auto IPriv = Privates.begin();
4791 for (
unsigned I = 0, E = ReductionOps.size(); I <
E; ++
I, ++IPriv, ++Idx) {
4792 auto RHSVar = cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[
I])->getDecl());
4796 auto LHSVar = cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[
I])->getDecl());
4800 QualType PrivTy = (*IPriv)->getType();
4808 auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
4810 CGF, OVE, RValue::get(CGF.
Builder.CreatePtrToInt(Ptr, CGF.
SizeTy)));
4815 IPriv = Privates.begin();
4816 auto ILHS = LHSExprs.begin();
4817 auto IRHS = RHSExprs.begin();
4818 for (
auto *E : ReductionOps) {
4819 if ((*IPriv)->getType()->isArrayType()) {
4821 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4822 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4824 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4835 Scope.ForceCleanup();
4841 const Expr *ReductionOp,
4842 const Expr *PrivateRef,
4847 auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
4848 auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
4850 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4908 if (SimpleReduction) {
4910 auto IPriv = Privates.begin();
4911 auto ILHS = LHSExprs.begin();
4912 auto IRHS = RHSExprs.begin();
4913 for (
auto *E : ReductionOps) {
4915 cast<DeclRefExpr>(*IRHS));
4925 auto Size = RHSExprs.size();
4926 for (
auto *E : Privates) {
4927 if (E->getType()->isVariablyModifiedType())
4931 llvm::APInt ArraySize(32, Size);
4936 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
4937 auto IPriv = Privates.begin();
4939 for (
unsigned I = 0, E = RHSExprs.size(); I <
E; ++
I, ++IPriv, ++Idx) {
4946 if ((*IPriv)->getType()->isVariablyModifiedType()) {
4964 LHSExprs, RHSExprs, ReductionOps);
4971 auto *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
4972 auto *ThreadId = getThreadID(CGF, Loc);
4973 auto *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
4979 CGF.
Builder.getInt32(RHSExprs.size()),
4980 ReductionArrayTySize,
4992 auto *SwInst = CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5001 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5010 auto &&CodeGen = [&
Privates, &LHSExprs, &RHSExprs, &ReductionOps](
5012 auto &RT = CGF.CGM.getOpenMPRuntime();
5013 auto IPriv = Privates.begin();
5014 auto ILHS = LHSExprs.begin();
5015 auto IRHS = RHSExprs.begin();
5016 for (
auto *E : ReductionOps) {
5017 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5018 cast<DeclRefExpr>(*IRHS));
5030 RCG.setAction(Action);
5033 CGF.EmitBranch(DefaultBB);
5040 auto *Case2BB = CGF.createBasicBlock(
".omp.reduction.case2");
5041 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5042 CGF.EmitBlock(Case2BB);
5044 auto &&AtomicCodeGen = [Loc, &
Privates, &LHSExprs, &RHSExprs, &ReductionOps](
5046 auto ILHS = LHSExprs.begin();
5047 auto IRHS = RHSExprs.begin();
5048 auto IPriv = Privates.begin();
5049 for (
auto *E : ReductionOps) {
5050 const Expr *XExpr =
nullptr;
5051 const Expr *EExpr =
nullptr;
5052 const Expr *UpExpr =
nullptr;
5054 if (
auto *BO = dyn_cast<BinaryOperator>(E)) {
5055 if (BO->getOpcode() == BO_Assign) {
5056 XExpr = BO->getLHS();
5057 UpExpr = BO->getRHS();
5061 auto *RHSExpr = UpExpr;
5064 if (
auto *ACO = dyn_cast<AbstractConditionalOperator>(
5065 RHSExpr->IgnoreParenImpCasts())) {
5068 RHSExpr = ACO->getCond();
5071 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5072 EExpr = BORHS->getRHS();
5073 BO = BORHS->getOpcode();
5077 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5078 auto &&AtomicRedGen = [BO, VD,
5080 const Expr *EExpr,
const Expr *UpExpr) {
5081 LValue X = CGF.EmitLValue(XExpr);
5084 E = CGF.EmitAnyExpr(EExpr);
5085 CGF.EmitOMPAtomicSimpleUpdateExpr(
5087 llvm::AtomicOrdering::Monotonic, Loc,
5088 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5090 PrivateScope.addPrivate(
5091 VD, [&CGF, VD, XRValue, Loc]() ->
Address {
5092 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5093 CGF.emitOMPSimpleStore(
5094 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5095 VD->getType().getNonReferenceType(), Loc);
5098 (void)PrivateScope.Privatize();
5099 return CGF.EmitAnyExpr(UpExpr);
5102 if ((*IPriv)->getType()->isArrayType()) {
5104 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5106 AtomicRedGen, XExpr, EExpr, UpExpr);
5109 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5113 const Expr *,
const Expr *) {
5114 auto &RT = CGF.CGM.getOpenMPRuntime();
5115 RT.emitCriticalRegion(
5116 CGF,
".atomic_reduction",
5123 if ((*IPriv)->getType()->isArrayType()) {
5124 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5125 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5129 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5147 AtomicRCG.setAction(Action);
5161 llvm::raw_svector_ostream Out(Buffer);
5180 Args.emplace_back(&Param);
5192 C.getPointerType(C.VoidPtrTy).castAs<
PointerType>());
5217 llvm::ConstantPointerNull::get(CGM.
VoidPtrTy),
5242 const Expr *ReductionOp,
5243 const Expr *LHS,
const Expr *RHS,
5244 const Expr *PrivateRef) {
5246 auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5247 auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5251 Args.emplace_back(&ParamInOut);
5252 Args.emplace_back(&ParamIn);
5282 C.getPointerType(C.VoidPtrTy).castAs<
PointerType>());
5290 C.getPointerType(C.VoidPtrTy).castAs<
PointerType>());
5299 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5300 cast<DeclRefExpr>(RHS));
5321 Args.emplace_back(&Param);
5333 C.getPointerType(C.VoidPtrTy).castAs<
PointerType>());
5355 ArrayRef<const Expr *> RHSExprs,
const OMPTaskDataTy &Data) {
5378 RD->completeDefinition();
5381 llvm::APInt ArraySize(64, Size);
5388 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5391 llvm::ConstantInt::get(CGM.
SizeTy, Cnt)};
5399 RCG.emitSharedLValue(CGF, Cnt);
5403 RCG.emitAggregateType(CGF, Cnt);
5406 std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);
5413 bool DelayedCreation = !!SizeVal;
5414 SizeValInChars = CGF.
Builder.CreateIntCast(SizeValInChars, CGM.
SizeTy,
5423 DelayedCreation = DelayedCreation || RCG.usesReductionInitializer(Cnt);
5429 : llvm::ConstantPointerNull::get(CGM.
VoidPtrTy);
5434 CGM, Loc, RCG, Cnt, Data.
ReductionOps[Cnt], LHSExprs[Cnt],
5439 if (DelayedCreation) {
5441 llvm::ConstantInt::get(CGM.
Int32Ty, 1,
true),
5449 CGF.
Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.
IntTy,
5451 llvm::ConstantInt::get(CGM.
IntTy, Size,
true),
5492 CGF.
Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.
IntTy,
5509 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
5512 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
5513 Region->emitUntiedSwitch(CGF);
5522 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
5538 if (CancelRegion == OMPD_parallel)
5539 CancelKind = CancelParallel;
5540 else if (CancelRegion == OMPD_for)
5541 CancelKind = CancelLoop;
5542 else if (CancelRegion == OMPD_sections)
5543 CancelKind = CancelSections;
5545 assert(CancelRegion == OMPD_taskgroup);
5546 CancelKind = CancelTaskgroup;
5558 if (
auto *OMPRegionInfo =
5562 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
5564 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
5575 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5593 if (
auto *OMPRegionInfo =
5595 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](
CodeGenFunction &CGF,
5597 auto &RT = CGF.CGM.getOpenMPRuntime();
5599 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
5602 auto *
Result = CGF.EmitRuntimeCall(
5607 auto *ExitBB = CGF.createBasicBlock(
".cancel.exit");
5608 auto *ContBB = CGF.createBasicBlock(
".cancel.continue");
5609 auto *Cmp = CGF.Builder.CreateIsNotNull(
Result);
5610 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5611 CGF.EmitBlock(ExitBB);
5614 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5615 CGF.EmitBranchThroughCleanup(CancelDest);
5616 CGF.EmitBlock(ContBB,
true);
5619 emitOMPIfClause(CGF, IfCond, ThenGen,
5632 unsigned &DeviceID,
unsigned &FileID,
5633 unsigned &LineNum) {
5640 assert(Loc.
isValid() &&
"Source location is expected to be always valid.");
5641 assert(Loc.
isFileID() &&
"Source location is expected to refer to a file.");
5644 assert(PLoc.isValid() &&
"Source location is expected to be always valid.");
5646 llvm::sys::fs::UniqueID
ID;
5647 if (llvm::sys::fs::getUniqueID(PLoc.getFilename(),
ID))
5648 llvm_unreachable(
"Source file with target region no longer exists!");
5650 DeviceID = ID.getDevice();
5651 FileID = ID.getFile();
5652 LineNum = PLoc.getLine();
5657 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5659 assert(!ParentName.empty() &&
"Invalid target region parent name!");
5661 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
5662 IsOffloadEntry, CodeGen);
5665 void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
5667 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5685 llvm::raw_svector_ostream OS(EntryFnName);
5686 OS <<
"__omp_offloading" << llvm::format(
"_%x", DeviceID)
5687 << llvm::format(
"_%x_", FileID) << ParentName <<
"_l" <<
Line;
5693 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
5696 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
5700 if (!IsOffloadEntry)
5715 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.
Int8PtrTy);
5718 OutlinedFnID =
new llvm::GlobalVariable(
5720 llvm::GlobalValue::PrivateLinkage,
5721 llvm::Constant::getNullValue(CGM.
Int8Ty),
".omp_offload.region_id");
5725 DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
5731 while (
auto *CS = dyn_cast_or_null<CompoundStmt>(Body))
5732 Body = CS->body_front();
5750 assert(!CGF.
getLangOpts().OpenMPIsDevice &&
"Clauses associated with the "
5751 "teams directive expected to be "
5752 "emitted only for the host!");
5762 auto NumTeams = CGF.
EmitScalarExpr(NumTeamsClause->getNumTeams(),
5764 return Bld.CreateIntCast(NumTeams, CGF.
Int32Ty,
5769 return Bld.getInt32(0);
5775 return Bld.getInt32(1);
5787 if (
auto *TeamsDir = dyn_cast_or_null<OMPTeamsDirective>(
5790 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
5793 return Bld.CreateIntCast(NumTeams, CGF.
Int32Ty,
5799 return Bld.getInt32(0);
5819 assert(!CGF.
getLangOpts().OpenMPIsDevice &&
"Clauses associated with the "
5820 "teams directive expected to be "
5821 "emitted only for the host!");
5841 llvm::Value *DefaultThreadLimitVal = Bld.getInt32(0);
5845 if (
const auto *ThreadLimitClause =
5848 auto ThreadLimit = CGF.
EmitScalarExpr(ThreadLimitClause->getThreadLimit(),
5850 ThreadLimitVal = Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
5854 if (
const auto *NumThreadsClause =
5861 Bld.CreateIntCast(NumThreads, CGF.
Int32Ty,
true);
5866 ThreadLimitVal = ThreadLimitVal
5867 ? Bld.CreateSelect(Bld.CreateICmpSLT(NumThreadsVal,
5869 NumThreadsVal, ThreadLimitVal)
5874 if (!ThreadLimitVal)
5875 ThreadLimitVal = DefaultThreadLimitVal;
5877 return ThreadLimitVal;
5890 if (
auto *TeamsDir = dyn_cast_or_null<OMPTeamsDirective>(
5893 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
5902 return CGF.
Builder.getInt32(0);
5914 class MappableExprsHandler {
5918 enum OpenMPOffloadMappingFlags {
5922 OMP_MAP_FROM = 0x02,
5925 OMP_MAP_ALWAYS = 0x04,
5928 OMP_MAP_DELETE = 0x08,
5931 OMP_MAP_IS_PTR = 0x10,
5936 OMP_MAP_FIRST_REF = 0x20,
5939 OMP_MAP_RETURN_PTR = 0x40,
5942 OMP_MAP_PRIVATE_PTR = 0x80,
5944 OMP_MAP_PRIVATE_VAL = 0x100,
5949 class BasePointerInfo {
5958 : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
5960 const ValueDecl *getDevicePtrDecl()
const {
return DevPtrDecl; }
5961 void setDevicePtrDecl(
const ValueDecl *D) { DevPtrDecl = D; }
5976 llvm::SmallPtrSet<const VarDecl *, 8> FirstPrivateDecls;
5985 llvm::Value *getExprTypeSize(
const Expr *E)
const {
5995 if (
const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
5997 OAE->getBase()->IgnoreParenImpCasts())
5998 .getCanonicalType();
6002 if (!OAE->getLength() && OAE->getColonLoc().isValid())
6007 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
6009 auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
6010 assert(ATy &&
"Expecting array type if not a pointer type.");
6011 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
6016 if (!OAE->getLength())
6022 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
6033 bool AddIsFirstFlag)
const {
6036 case OMPC_MAP_alloc:
6037 case OMPC_MAP_release:
6047 Bits = OMP_MAP_FROM;
6049 case OMPC_MAP_tofrom:
6050 Bits = OMP_MAP_TO | OMP_MAP_FROM;
6052 case OMPC_MAP_delete:
6053 Bits = OMP_MAP_DELETE;
6056 llvm_unreachable(
"Unexpected map type!");
6060 Bits |= OMP_MAP_IS_PTR;
6062 Bits |= OMP_MAP_FIRST_REF;
6063 if (MapTypeModifier == OMPC_MAP_always)
6064 Bits |= OMP_MAP_ALWAYS;
6070 bool isFinalArraySectionExpression(
const Expr *E)
const {
6078 if (OASE->getColonLoc().isInvalid())
6081 auto *
Length = OASE->getLength();
6088 OASE->getBase()->IgnoreParenImpCasts())
6089 .getCanonicalType();
6090 if (
auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
6091 return ATy->getSize().getSExtValue() != 1;
6099 llvm::APSInt ConstLength;
6103 return ConstLength.getSExtValue() != 1;
6110 void generateInfoForComponentList(
6113 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
6114 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
6115 bool IsFirstComponentList)
const {
6231 bool IsCaptureFirstInfo = IsFirstComponentList;
6234 auto CI = Components.rbegin();
6235 auto CE = Components.rend();
6240 bool IsExpressionFirstInfo =
true;
6243 if (
auto *ME = dyn_cast<MemberExpr>(I->getAssociatedExpression())) {
6250 BP = CGF.
EmitLValue(cast<DeclRefExpr>(I->getAssociatedExpression()))
6257 I->getAssociatedDeclaration()->getType().getNonReferenceType();
6270 for (; I != CE; ++
I) {
6271 auto Next = std::next(I);
6279 bool IsFinalArraySection =
6280 isFinalArraySectionExpression(I->getAssociatedExpression());
6294 if (
Next == CE || IsPointer || IsFinalArraySection) {
6298 assert((
Next == CE ||
6299 isa<MemberExpr>(
Next->getAssociatedExpression()) ||
6300 isa<ArraySubscriptExpr>(
Next->getAssociatedExpression()) ||
6301 isa<OMPArraySectionExpr>(
Next->getAssociatedExpression())) &&
6302 "Unexpected expression");
6304 auto *LB = CGF.
EmitLValue(I->getAssociatedExpression()).getPointer();
6305 auto *Size = getExprTypeSize(I->getAssociatedExpression());
6311 if (isa<MemberExpr>(I->getAssociatedExpression()) &&
6312 I->getAssociatedDeclaration()->getType()->isReferenceType()) {
6313 auto *LI = cast<llvm::LoadInst>(LB);
6314 auto *RefAddr = LI->getPointerOperand();
6316 BasePointers.push_back(BP);
6317 Pointers.push_back(RefAddr);
6319 Types.push_back(getMapTypeBits(
6321 !IsExpressionFirstInfo, IsCaptureFirstInfo));
6322 IsExpressionFirstInfo =
false;
6323 IsCaptureFirstInfo =
false;
6328 BasePointers.push_back(BP);
6329 Pointers.push_back(LB);
6330 Sizes.push_back(Size);
6336 Types.push_back(getMapTypeBits(MapType, MapTypeModifier,
6337 !IsExpressionFirstInfo,
6338 IsCaptureFirstInfo));
6341 if (IsFinalArraySection)
6348 IsExpressionFirstInfo =
false;
6349 IsCaptureFirstInfo =
false;
6359 unsigned CurrentModifiers) {
6366 return MappableExprsHandler::OMP_MAP_PRIVATE_PTR |
6367 MappableExprsHandler::OMP_MAP_TO;
6370 return CurrentModifiers;
6375 : CurDir(Dir), CGF(CGF) {
6378 for (
const auto *D : C->varlists())
6379 FirstPrivateDecls.insert(
6383 for (
auto L : C->component_lists())
6384 DevPointersMap[L.first].push_back(L.second);
6391 void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
6392 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
6393 MapFlagsArrayTy &Types)
const {
6394 BasePointers.clear();
6401 enum ReturnPointerKind {
6409 RPK_MemberReference,
6414 ReturnPointerKind ReturnDevicePointer;
6418 ReturnDevicePointer(RPK_None) {}
6422 ReturnPointerKind ReturnDevicePointer)
6423 : Components(Components), MapType(MapType),
6424 MapTypeModifier(MapTypeModifier),
6425 ReturnDevicePointer(ReturnDevicePointer) {}
6431 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
6435 auto &&InfoGen = [&Info](
6439 MapInfo::ReturnPointerKind ReturnDevicePointer) {
6442 Info[VD].push_back({L, MapType, MapModifier, ReturnDevicePointer});
6446 for (
auto *C : this->CurDir.getClausesOfKind<
OMPMapClause>())
6447 for (
auto L : C->component_lists())
6448 InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifier(),
6450 for (
auto *C : this->CurDir.getClausesOfKind<
OMPToClause>())
6451 for (
auto L : C->component_lists())
6454 for (
auto *C : this->CurDir.getClausesOfKind<
OMPFromClause>())
6455 for (
auto L : C->component_lists())
6465 for (
auto L : C->component_lists()) {
6466 assert(!L.second.empty() &&
"Not expecting empty list of components!");
6467 const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
6468 VD = cast<ValueDecl>(VD->getCanonicalDecl());
6469 auto *IE = L.second.back().getAssociatedExpression();
6473 auto It = Info.find(isa<MemberExpr>(IE) ?
nullptr : VD);
6477 if (It != Info.end()) {
6478 auto CI = std::find_if(
6479 It->second.begin(), It->second.end(), [VD](
const MapInfo &MI) {
6480 return MI.Components.back().getAssociatedDeclaration() == VD;
6484 if (CI != It->second.end()) {
6485 CI->ReturnDevicePointer = isa<MemberExpr>(IE)
6486 ? (VD->getType()->isReferenceType()
6487 ? MapInfo::RPK_MemberReference
6488 : MapInfo::RPK_Member)
6489 : MapInfo::RPK_Base;
6501 BasePointers.push_back({Ptr, VD});
6502 Pointers.push_back(Ptr);
6503 Sizes.push_back(llvm::Constant::getNullValue(this->CGF.
SizeTy));
6504 Types.push_back(OMP_MAP_RETURN_PTR | OMP_MAP_FIRST_REF);
6507 for (
auto &M : Info) {
6510 bool IsFirstComponentList =
true;
6511 for (MapInfo &L : M.second) {
6512 assert(!L.Components.empty() &&
6513 "Not expecting declaration with no component lists.");
6516 unsigned CurrentBasePointersIdx = BasePointers.size();
6518 this->generateInfoForComponentList(L.MapType, L.MapTypeModifier,
6519 L.Components, BasePointers, Pointers,
6520 Sizes, Types, IsFirstComponentList);
6524 if (IsFirstComponentList &&
6525 L.ReturnDevicePointer != MapInfo::RPK_None) {
6529 if (L.ReturnDevicePointer != MapInfo::RPK_Base) {
6530 ++CurrentBasePointersIdx;
6531 if (L.ReturnDevicePointer == MapInfo::RPK_MemberReference)
6532 ++CurrentBasePointersIdx;
6534 assert(BasePointers.size() > CurrentBasePointersIdx &&
6535 "Unexpected number of mapped base pointers.");
6537 auto *RelevantVD = L.Components.back().getAssociatedDeclaration();
6538 assert(RelevantVD &&
6539 "No relevant declaration related with device pointer??");
6541 BasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
6542 Types[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PTR;
6544 IsFirstComponentList =
false;
6553 MapBaseValuesArrayTy &BasePointers,
6554 MapValuesArrayTy &Pointers,
6555 MapValuesArrayTy &Sizes,
6556 MapFlagsArrayTy &Types)
const {
6558 "Not expecting to generate map info for a variable array type!");
6560 BasePointers.clear();
6567 bool IsFirstComponentList =
true;
6579 auto It = DevPointersMap.find(VD);
6580 if (It != DevPointersMap.end()) {
6581 for (
auto L : It->second) {
6582 generateInfoForComponentList(
6584 BasePointers, Pointers, Sizes, Types, IsFirstComponentList);
6585 IsFirstComponentList =
false;
6589 }
else if (DevPointersMap.count(VD)) {
6590 BasePointers.push_back({Arg, VD});
6591 Pointers.push_back(Arg);
6593 Types.push_back(OMP_MAP_PRIVATE_VAL | OMP_MAP_FIRST_REF);
6598 for (
auto *C : this->CurDir.getClausesOfKind<
OMPMapClause>())
6599 for (
auto L : C->decl_component_lists(VD)) {
6600 assert(L.first == VD &&
6601 "We got information for the wrong declaration??");
6602 assert(!L.second.empty() &&
6603 "Not expecting declaration with no component lists.");
6604 generateInfoForComponentList(C->getMapType(), C->getMapTypeModifier(),
6605 L.second, BasePointers, Pointers, Sizes,
6606 Types, IsFirstComponentList);
6607 IsFirstComponentList =
false;
6617 MapBaseValuesArrayTy &CurBasePointers,
6618 MapValuesArrayTy &CurPointers,
6619 MapValuesArrayTy &CurSizes,
6620 MapFlagsArrayTy &CurMapTypes) {
6624 CurBasePointers.push_back(CV);
6625 CurPointers.push_back(CV);
6629 CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
6631 CurBasePointers.push_back(CV);
6632 CurPointers.push_back(CV);
6636 CurMapTypes.push_back(OMP_MAP_PRIVATE_VAL);
6641 CurMapTypes.push_back(0u);
6642 CurSizes.push_back(llvm::Constant::getNullValue(CGF.
SizeTy));
6646 CurBasePointers.push_back(CV);
6647 CurPointers.push_back(CV);
6656 CurMapTypes.push_back(ElementType->isAggregateType()
6657 ? (OMP_MAP_TO | OMP_MAP_FROM)
6663 CurMapTypes.back() =
6664 adjustMapModifiersForPrivateClauses(CI, CurMapTypes.back());
6668 CurMapTypes.back() |= OMP_MAP_FIRST_REF;
6675 OMP_DEVICEID_UNDEF = -1,
6688 CGOpenMPRuntime::TargetDataInfo &Info) {
6689 auto &CGM = CGF.
CGM;
6693 Info.clearArrayInfo();
6694 Info.NumberOfPtrs = BasePointers.size();
6696 if (Info.NumberOfPtrs) {
6699 bool hasRuntimeEvaluationCaptureSize =
false;
6700 for (
auto *
S : Sizes)
6701 if (!isa<llvm::Constant>(
S)) {
6702 hasRuntimeEvaluationCaptureSize =
true;
6706 llvm::APInt PointerNumAP(32, Info.NumberOfPtrs,
true);
6711 Info.BasePointersArray =
6713 Info.PointersArray =
6719 if (hasRuntimeEvaluationCaptureSize) {
6720 QualType SizeArrayType = Ctx.getConstantArrayType(
6729 for (
auto S : Sizes)
6730 ConstSizes.push_back(cast<llvm::Constant>(
S));
6732 auto *SizesArrayInit = llvm::ConstantArray::get(
6733 llvm::ArrayType::get(CGM.
SizeTy, ConstSizes.size()), ConstSizes);
6734 auto *SizesArrayGbl =
new llvm::GlobalVariable(
6735 CGM.
getModule(), SizesArrayInit->getType(),
6736 true, llvm::GlobalValue::PrivateLinkage,
6737 SizesArrayInit,
".offload_sizes");
6738 SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6739 Info.SizesArray = SizesArrayGbl;
6744 llvm::Constant *MapTypesArrayInit =
6745 llvm::ConstantDataArray::get(CGF.
Builder.getContext(), MapTypes);
6746 auto *MapTypesArrayGbl =
new llvm::GlobalVariable(
6747 CGM.
getModule(), MapTypesArrayInit->getType(),
6748 true, llvm::GlobalValue::PrivateLinkage,
6749 MapTypesArrayInit,
".offload_maptypes");
6750 MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6751 Info.MapTypesArray = MapTypesArrayGbl;
6753 for (
unsigned i = 0; i < Info.NumberOfPtrs; ++i) {
6756 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
6757 Info.BasePointersArray, 0, i);
6759 BP, BPVal->getType()->getPointerTo(0));
6760 Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
6763 if (Info.requiresDevicePointerInfo())
6764 if (
auto *DevVD = BasePointers[i].getDevicePtrDecl())
6765 Info.CaptureDeviceAddrMap.insert(std::make_pair(DevVD, BPAddr));
6769 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
6770 Info.PointersArray, 0, i);
6772 P, PVal->getType()->getPointerTo(0));
6773 Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
6776 if (hasRuntimeEvaluationCaptureSize) {
6778 llvm::ArrayType::get(CGM.
SizeTy, Info.NumberOfPtrs),
6782 Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType()));
6795 llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
6796 auto &CGM = CGF.
CGM;
6797 if (Info.NumberOfPtrs) {
6798 BasePointersArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
6799 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
6800 Info.BasePointersArray,
6802 PointersArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
6803 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
6807 SizesArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
6808 llvm::ArrayType::get(CGM.
SizeTy, Info.NumberOfPtrs), Info.SizesArray,
6810 MapTypesArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
6811 llvm::ArrayType::get(CGM.
Int32Ty, Info.NumberOfPtrs),
6816 BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
6817 PointersArrayArg = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
6818 SizesArrayArg = llvm::ConstantPointerNull::get(CGM.
SizeTy->getPointerTo());
6820 llvm::ConstantPointerNull::get(CGM.
Int32Ty->getPointerTo());
6828 const Expr *IfCond,
const Expr *Device,
6829 ArrayRef<llvm::Value *> CapturedVars) {
6833 assert(OutlinedFn &&
"Invalid outlined function!");
6850 MappableExprsHandler MEHandler(D, CGF);
6854 auto CV = CapturedVars.begin();
6857 CI != CE; ++CI, ++RI, ++CV) {
6861 CurBasePointers.clear();
6862 CurPointers.clear();
6864 CurMapTypes.clear();
6869 CurBasePointers.push_back(*CV);
6870 CurPointers.push_back(*CV);
6871 CurSizes.push_back(CGF.
getTypeSize(RI->getType()));
6873 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_PRIVATE_VAL |
6874 MappableExprsHandler::OMP_MAP_FIRST_REF);
6878 MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
6879 CurSizes, CurMapTypes);
6880 if (CurBasePointers.empty())
6881 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
6882 CurPointers, CurSizes, CurMapTypes);
6885 assert(!CurBasePointers.empty() &&
"Non-existing map pointer for capture!");
6886 assert(CurBasePointers.size() == CurPointers.size() &&
6887 CurBasePointers.size() == CurSizes.size() &&
6888 CurBasePointers.size() == CurMapTypes.size() &&
6889 "Inconsistent map information sizes!");
6893 KernelArgs.push_back(*CurBasePointers.front());
6895 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
6896 Pointers.append(CurPointers.begin(), CurPointers.end());
6897 Sizes.append(CurSizes.begin(), CurSizes.end());
6898 MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
6902 auto OffloadErrorQType =
6903 Ctx.getIntTypeForBitwidth(32,
true);
6911 auto &&ThenGen = [&BasePointers, &Pointers, &Sizes, &MapTypes, Device,
6912 OutlinedFnID, OffloadError,
6914 auto &RT = CGF.CGM.getOpenMPRuntime();
6932 assert(OutlinedFnID &&
"Invalid outlined function ID!");
6937 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
6940 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
6943 llvm::Value *PointerNum = CGF.Builder.getInt32(BasePointers.size());
6984 assert(NumThreads &&
"Thread limit expression should be available along "
6985 "with number of teams.");
6987 DeviceID, OutlinedFnID,
6992 Return = CGF.EmitRuntimeCall(
6996 DeviceID, OutlinedFnID,
7004 CGF.EmitStoreOfScalar(Return, OffloadError);
7019 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
7033 auto Failed = CGF.
Builder.CreateIsNotNull(OffloadErrorVal);
7034 CGF.
Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
7037 CGF.
Builder.CreateCall(OutlinedFn, KernelArgs);
7044 StringRef ParentName) {
7049 bool requiresDeviceCodegen =
7050 isa<OMPExecutableDirective>(
S) &&
7052 cast<OMPExecutableDirective>(S)->getDirectiveKind());
7054 if (requiresDeviceCodegen) {
7055 auto &E = *cast<OMPExecutableDirective>(
S);
7069 case Stmt::OMPTargetDirectiveClass:
7070 CodeGenFunction::EmitOMPTargetDeviceFunction(
7071 CGM, ParentName, cast<OMPTargetDirective>(*S));
7073 case Stmt::OMPTargetParallelDirectiveClass:
7074 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
7075 CGM, ParentName, cast<OMPTargetParallelDirective>(*S));
7077 case Stmt::OMPTargetTeamsDirectiveClass:
7078 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
7079 CGM, ParentName, cast<OMPTargetTeamsDirective>(*S));
7082 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
7088 if (!E->hasAssociatedStmt())
7092 cast<CapturedStmt>(E->getAssociatedStmt())->getCapturedStmt(),
7098 if (
auto *L = dyn_cast<LambdaExpr>(S))
7107 auto &FD = *cast<FunctionDecl>(GD.
getDecl());
7132 for (
auto *Ctor : RD->ctors()) {
7133 StringRef ParentName =
7137 auto *Dtor = RD->getDestructor();
7139 StringRef ParentName =
7152 if (isa<FunctionDecl>(VD))
7177 auto *RTLoc = emitUpdateLocation(CGF, Loc);
7183 CGF.
Builder.getInt32(CapturedVars.size()),
7186 RealArgs.append(std::begin(Args), std::end(Args));
7187 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
7194 const Expr *NumTeams,
7195 const Expr *ThreadLimit,
7200 auto *RTLoc = emitUpdateLocation(CGF, Loc);
7215 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
7234 auto &&BeginThenGen = [&D, Device, &Info, &CodeGen](
CodeGenFunction &CGF,
7243 MappableExprsHandler MCHandler(D, CGF);
7244 MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
7254 SizesArrayArg, MapTypesArrayArg, Info);
7259 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
7262 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
7265 auto *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
7268 DeviceID, PointerNum, BasePointersArrayArg,
7269 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
7270 auto &RT = CGF.CGM.getOpenMPRuntime();
7276 if (!Info.CaptureDeviceAddrMap.empty())
7282 assert(Info.isValid() &&
"Invalid data environment closing arguments.");
7289 SizesArrayArg, MapTypesArrayArg, Info);
7294 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
7297 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
7300 auto *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
7303 DeviceID, PointerNum, BasePointersArrayArg,
7304 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
7305 auto &RT = CGF.CGM.getOpenMPRuntime();
7313 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](
CodeGenFunction &CGF,
7315 if (!Info.CaptureDeviceAddrMap.empty()) {
7316 CodeGen.setAction(NoPrivAction);
7326 emitOMPIfClause(CGF, IfCond, BeginThenGen, BeginElseGen);
7334 if (Info.CaptureDeviceAddrMap.empty()) {
7335 CodeGen.setAction(NoPrivAction);
7340 emitOMPIfClause(CGF, IfCond, EndThenGen, EndElseGen);
7349 const Expr *Device) {
7353 assert((isa<OMPTargetEnterDataDirective>(D) ||
7354 isa<OMPTargetExitDataDirective>(D) ||
7355 isa<OMPTargetUpdateDirective>(D)) &&
7356 "Expecting either target enter, exit data, or update directives.");
7367 MappableExprsHandler MEHandler(D, CGF);
7368 MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
7380 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
7383 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
7386 auto *PointerNum = CGF.Builder.getInt32(BasePointers.size());
7392 auto &RT = CGF.CGM.getOpenMPRuntime();
7398 llvm_unreachable(
"Unexpected standalone target data directive.");
7400 case OMPD_target_enter_data:
7403 case OMPD_target_exit_data:
7406 case OMPD_target_update:
7410 CGF.EmitRuntimeCall(RT.createRuntimeFunction(RTLFn), OffloadingArgs);
7418 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
7429 struct ParamAttrTy {
7431 llvm::APSInt StrideOrArg;
7432 llvm::APSInt Alignment;
7437 ArrayRef<ParamAttrTy> ParamAttrs) {
7466 if (
auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
7467 if (ParamAttrs[Offset].
Kind == Vector)
7473 if (ParamAttrs[I + Offset].
Kind == Vector) {
7490 const llvm::APSInt &VLENVal,
7491 ArrayRef<ParamAttrTy> ParamAttrs,
7492 OMPDeclareSimdDeclAttr::BranchStateTy
State) {
7495 unsigned VecRegSize;
7497 ISADataTy ISAData[] = {
7513 case OMPDeclareSimdDeclAttr::BS_Undefined:
7514 Masked.push_back(
'N');
7515 Masked.push_back(
'M');
7517 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
7518 Masked.push_back(
'N');
7520 case OMPDeclareSimdDeclAttr::BS_Inbranch:
7521 Masked.push_back(
'M');
7524 for (
auto Mask : Masked) {
7525 for (
auto &Data : ISAData) {
7527 llvm::raw_svector_ostream Out(Buffer);
7528 Out <<
"_ZGV" << Data.ISA << Mask;
7530 Out << llvm::APSInt::getUnsigned(Data.VecRegSize /
7534 for (
auto &ParamAttr : ParamAttrs) {
7535 switch (ParamAttr.Kind){
7536 case LinearWithVarStride:
7537 Out <<
's' << ParamAttr.StrideOrArg;
7541 if (!!ParamAttr.StrideOrArg)
7542 Out << ParamAttr.StrideOrArg;
7551 if (!!ParamAttr.Alignment)
7552 Out <<
'a' << ParamAttr.Alignment;
7554 Out <<
'_' << Fn->getName();
7555 Fn->addFnAttr(Out.str());
7561 llvm::Function *Fn) {
7565 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
7566 if (isa<CXXMethodDecl>(FD))
7567 ParamPositions.insert({FD, 0});
7568 unsigned ParamPos = ParamPositions.size();
7570 ParamPositions.insert({
P->getCanonicalDecl(), ParamPos});
7576 for (
auto *E :
Attr->uniforms()) {
7579 if (isa<CXXThisExpr>(E))
7580 Pos = ParamPositions[FD];
7582 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
7584 Pos = ParamPositions[PVD];
7586 ParamAttrs[Pos].Kind = Uniform;
7589 auto NI =
Attr->alignments_begin();
7590 for (
auto *E :
Attr->aligneds()) {
7594 if (isa<CXXThisExpr>(E)) {
7595 Pos = ParamPositions[FD];
7598 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
7600 Pos = ParamPositions[PVD];
7601 ParmTy = PVD->getType();
7603 ParamAttrs[Pos].Alignment =
7604 (*NI) ? (*NI)->EvaluateKnownConstInt(C)
7605 : llvm::APSInt::getUnsigned(
7611 auto SI =
Attr->steps_begin();
7612 auto MI =
Attr->modifiers_begin();
7613 for (
auto *E :
Attr->linears()) {
7616 if (isa<CXXThisExpr>(E))
7617 Pos = ParamPositions[FD];
7619 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
7621 Pos = ParamPositions[PVD];
7623 auto &ParamAttr = ParamAttrs[Pos];
7624 ParamAttr.Kind = Linear;
7626 if (!(*SI)->EvaluateAsInt(ParamAttr.StrideOrArg, C,
7628 if (
auto *DRE = cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
7629 if (
auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
7630 ParamAttr.Kind = LinearWithVarStride;
7631 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
7632 ParamPositions[StridePVD->getCanonicalDecl()]);
7640 llvm::APSInt VLENVal;
7641 if (
const Expr *VLEN =
Attr->getSimdlen())
7642 VLENVal = VLEN->EvaluateKnownConstInt(C);
7643 OMPDeclareSimdDeclAttr::BranchStateTy
State =
Attr->getBranchState();
7644 if (CGM.
getTriple().getArch() == llvm::Triple::x86 ||
7645 CGM.
getTriple().getArch() == llvm::Triple::x86_64)
7654 static const int DoacrossFinArgs = 2;
7663 assert(CallArgs.size() == DoacrossFinArgs);
7664 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
7700 enum { LowerFD = 0, UpperFD, StrideFD };
7720 llvm::ConstantInt::getSigned(CGM.
Int32Ty, 1),
7726 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
7730 llvm::makeArrayRef(FiniArgs));
7740 CounterVal->
getType(), Int64Ty,
7744 llvm::Value *Args[] = {emitUpdateLocation(CGF, C->getLocStart()),
7745 getThreadID(CGF, C->getLocStart()),
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
void pushTerminate()
Push a terminate handler on the stack.
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, llvm::Type *BaseLVType, CharUnits BaseLVAlignment, llvm::Value *Addr)
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
StmtClass getStmtClass() const
CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
static llvm::Value * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
FunctionDecl - An instance of this class is created to represent a function declaration or definition...
llvm::IntegerType * IntTy
int
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
External linkage, which indicates that the entity can be referred to from other translation units...
Parameter for captured context.
QualType TgtDeviceImageQTy
struct __tgt_device_image{ void *ImageStart; // Pointer to the target code start. ...
PointerType - C99 6.7.5.1 - Pointer Declarators.
Scheduling data for loop-based OpenMP directives.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
A (possibly-)qualified type.
llvm::Value * getPointer() const
CodeGenTypes & getTypes()
unsigned getColumn() const
Return the presumed column number of this location.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
Expr * getNumIterations() const
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, unsigned IVSize, bool IVSigned, bool Ordered, Address IL, Address LB, Address UB, Address ST, llvm::Value *Chunk=nullptr)
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after...
llvm::Module & getModule() const
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
RecordDecl * buildImplicitRecord(StringRef Name, RecordDecl::TagKind TK=TTK_Struct) const
Create a new implicit TU-level CXXRecordDecl or RecordDecl declaration.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::SmallPtrSet< const VarDecl *, 4 > ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
The standard implementation of ConstantInitBuilder used in Clang.
Stmt - This represents one statement.
llvm::ConstantInt * getSize(CharUnits N)
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
QualType getTgtBinaryDescriptorQTy()
Returns __tgt_bin_desc type.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
bool isRecordType() const
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
SmallVector< const Expr *, 4 > LastprivateCopies
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
Address getAddress() const
llvm::Constant * getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr, IdentFieldIndex Field, const llvm::Twine &Name="")
Call to void __kmpc_threadprivate_register( ident_t *, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);.
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
RecordDecl * KmpTaskTQTyRD
The base class of the type hierarchy.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
QualType getRecordType(const RecordDecl *Decl) const
std::unique_ptr< llvm::MemoryBuffer > Buffer
llvm::Value * PointersArray
The array of section pointers passed to the runtime library.
virtual void completeDefinition()
completeDefinition - Notes that the definition of this type is now complete.
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, bool forPointeeType=false)
const LangOptions & getLangOpts() const
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and and emit all target regions found along the way.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
static CharUnits getOffsetOfIdentField(IdentFieldIndex Field)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS...
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
OpenMPSchedType
Schedule types for 'omp for' loops (these enumerators are taken from the enum sched_type in kmp...
SmallVector< const Expr *, 4 > ReductionCopies
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
VarDecl - An instance of this class is created to represent a variable declaration or definition...
Objects with "hidden" visibility are not seen by the dynamic linker.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
Extra information about a function prototype.
field_iterator field_begin() const
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata...
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
OpenMPDirectiveKind getDirectiveKind() const
Struct that keeps all the relevant information that should be kept throughout a 'target data' region...
QualType getTgtOffloadEntryQTy()
Returns __tgt_offload_entry type.
SmallVector< const Expr *, 4 > PrivateVars
RecordDecl - Represents a struct/union/class.
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
Source[4] in Fortran, do not use for C++.
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Expr * getSizeExpr() const
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Call to void *__kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid, void *data...
llvm::IntegerType * Int64Ty
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc)
Emit flush of the variables specified in 'omp flush' directive.
bool isReferenceType() const
SmallVector< const Expr *, 4 > LastprivateVars
QualType getReturnType() const
FieldDecl - An instance of this class is created by Sema::ActOnField to represent a member of a struc...
bool isAnyPointerType() const
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr)
llvm::IntegerType * SizeTy
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, bool IsLowerBound=true)
static void emitOffloadingArrays(CodeGenFunction &CGF, MappableExprsHandler::MapBaseValuesArrayTy &BasePointers, MappableExprsHandler::MapValuesArrayTy &Pointers, MappableExprsHandler::MapValuesArrayTy &Sizes, MappableExprsHandler::MapFlagsArrayTy &MapTypes, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the arrays used to pass the captures and map information to the offloading runtime library...
llvm::CallInst * EmitRuntimeCall(llvm::Value *callee, const Twine &name="")
void startDefinition()
Starts the definition of this tag declaration.
This represents clause 'map' in the '#pragma omp ...' directives.
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable. ...
This represents clause 'to' in the '#pragma omp ...' directives.
static CharUnits Zero()
Zero - Construct a CharUnits quantity of zero.
llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee, ArrayRef< llvm::Value * > Args, const Twine &Name="")
Emits a call or invoke instruction to the given function, depending on the current state of the EH st...
clang::CharUnits operator*(clang::CharUnits::QuantityType Scale, const clang::CharUnits &CU)
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
Expr * getCounterValue()
Get the loop counter value.
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
const Decl * getDecl() const
unsigned size() const
Return number of entries defined so far.
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, LValueBaseInfo BaseInfo=LValueBaseInfo(AlignmentSource::Type), llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, Address IL, Address LB, Address UB, Address ST, llvm::Value *Chunk=nullptr)
Call the appropriate runtime routine to initialize it before start of loop.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
QualType TgtOffloadEntryQTy
Type struct __tgt_offload_entry{ void *addr; // Pointer to the offload entry info.
CharUnits getAlignment() const
static CharUnits getIdentAlign(CodeGenModule &CGM)
OpenMPScheduleClauseKind Schedule
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
const Qualifiers & getQuals() const
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
RAII for correct setting/restoring of CapturedStmtInfo.
llvm::PointerType * VoidPtrTy
String describing the source location.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
void actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action)
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
OpenMPScheduleClauseModifier M2
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind...
SmallVector< const Expr *, 4 > PrivateCopies
RecordDecl * getDecl() const
CharUnits getPointerSize() const
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
void EmitAggregateAssign(Address DestPtr, Address SrcPtr, QualType EltTy)
EmitAggregateCopy - Emit an aggregate assignment.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Scope - A scope is a transient data structure that is used while parsing the program.
llvm::PointerType * VoidPtrPtrTy
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
SmallVector< const Expr *, 4 > FirstprivateCopies
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static int addMonoNonMonoModifier(OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitOffloadingArraysArgument(CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg, llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the arguments to be passed to the runtime library based on the arrays of pointers, sizes and map types.
Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
virtual llvm::Value * emitTeamsOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
unsigned getLine() const
Return the presumed line number of this location.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
SmallVector< const Expr *, 4 > ReductionOps
This represents clause 'is_device_ptr' in the '#pragma omp ...' directives.
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp", llvm::Value *ArraySize=nullptr)
CreateTempAlloca - This creates an alloca and inserts it into the entry block if ArraySize is nullptr...
detail::InMemoryDirectory::const_iterator I
llvm::Value * EmitCheckedInBoundsGEP(llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
SmallVector< const Expr *, 4 > ReductionVars
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, ArrayRef< const Expr * > PrivateVars, ArrayRef< const Expr * > FirstprivateVars, ArrayRef< const Expr * > LastprivateVars, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables...
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
llvm::Value * emitReductionFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
This represents clause 'from' in the '#pragma omp ...' directives.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
virtual llvm::Value * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
OpenMPDependClauseKind getDependencyKind() const
Get dependency type.
OpenMP 4.0 [2.4, Array Sections].
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
CanQualType getCanonicalTypeUnqualified() const
Describes the capture of either a variable, or 'this', or variable-length array type.
llvm::Constant * createForStaticInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
bool addPrivate(const VarDecl *LocalVD, llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
Class intended to support codegen of all kind of the reduction clauses.
llvm::Constant * createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned...
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
QualType getTgtDeviceImageQTy()
Returns __tgt_device_image type.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
KmpTaskTFields
Indexes of fields for type kmp_task_t.
static TypeEvaluationKind getEvaluationKind(QualType T)
hasAggregateLLVMType - Return true if the specified AST type will map into an aggregate LLVM type or ...
llvm::Value * getPointer() const
ValueDecl - Represent the declaration of a variable (in which case it is an lvalue) a function (in wh...
Expr - This represents one expression.
Allow any unmodeled side effect.
virtual llvm::Value * emitParallelOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
void loadOffloadInfoMetadata()
Loads all the offload entries information from the host IR metadata.
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
static llvm::Value * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Value *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info)
Emit the target data mapping code associated with D.
static llvm::Function * createOffloadingBinaryDescriptorFunction(CodeGenModule &CGM, StringRef Name, const RegionCodeGenTy &Codegen)
Create a Ctor/Dtor-like function whose body is emitted through Codegen.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::Constant *ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, unsigned IVSize, bool Ordered, Address IL, Address LB, Address UB, Address ST, llvm::Value *Chunk)
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD)
Checks if destructor function is required to be generated.
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
const ParmVarDecl * getParamDecl(unsigned i) const
ASTContext & getContext() const
llvm::BasicBlock * getBlock() const
void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, llvm::Constant *Addr, llvm::Constant *ID, int32_t Flags)
Register target region entry.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, LValueBaseInfo BaseInfo=LValueBaseInfo(AlignmentSource::Type), llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
llvm::IntegerType * Int32Ty
static OMPLinearClause * CreateEmpty(const ASTContext &C, unsigned NumVars)
Creates an empty clause with the place for NumVars variables.
SmallVector< const Expr *, 4 > FirstprivateVars
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Represents an unpacked "presumed" location which can be presented to the user.
void Emit(CodeGenFunction &CGF, Flags) override
Emit the cleanup.
unsigned Map[FirstTargetAddressSpace]
The type of a lookup table which maps from language-specific address spaces to target-specific ones...
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
CGOpenMPRuntime(CodeGenModule &CGM)
OpenMPProcBindClauseKind
OpenMP attributes for 'proc_bind' clause.
virtual llvm::Function * emitRegistrationFunction()
Creates the offloading descriptor in the event any target region was emitted in the current module an...
llvm::Constant * createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned...
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc)
Emit code for 'taskwait' directive.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
GlobalDecl - represents a global declaration.
const Expr * getAnyInitializer() const
getAnyInitializer - Get the initializer for this variable, no matter which declaration it is attached...
bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum) const
Return true if a target region entry with the provided information exists.
SourceLocation getLocStart() const
Returns starting location of directive kind.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
static llvm::Value * emitNumThreadsForTargetDirective(CGOpenMPRuntime &OMPRuntime, CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the number of threads for a target directive.
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
QualType TgtBinaryDescriptorQTy
struct __tgt_bin_desc{ int32_t NumDevices; // Number of devices supported.
The l-value was considered opaque, so the alignment was determined from a type.
const MatchFinder::MatchFinderOptions & Options
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
static llvm::Value * emitNumTeamsForTargetDirective(CGOpenMPRuntime &OMPRuntime, CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the number of teams for a target directive.
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
This captures a statement into a function.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
const char * getFilename() const
Return the presumed filename of this location.
static with chunk adjustment (e.g., simd)
ASTContext & getContext() const
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup...
void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
Encodes a location in the source.
CharUnits getPointerAlign() const
unsigned getNumParams() const
getNumParams - Return the number of parameters this function must have based on its FunctionType...
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
llvm::Value * MapTypesArray
The array of map types passed to the runtime library.
This represents '#pragma omp declare reduction ...' directive.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams...
This is a basic class for representing single OpenMP executable directive.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
bool isValid() const
Return true if this is a valid SourceLocation object.
Expr * getLowerBoundVariable() const
Lower bound for 'ordered' versions.
ASTContext & getASTContext() const LLVM_READONLY
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
OpenMPDirectiveKind
OpenMP directives.
Set if the nonmonotonic schedule modifier was present.
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
ArrayRef< ParmVarDecl * > parameters() const
Target region entries related.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
virtual void Enter(CodeGenFunction &CGF)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
const LangOptions & getLangOpts() const
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
void setAction(PrePostActionTy &Action) const
const T * castAs() const
Member-template castAs<specific type>.
lookup_result lookup(DeclarationName Name) const
lookup - Find the declarations (if any) with the given Name in this context.
Expr * getUpperBoundVariable() const
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
virtual void emitProcBindClause(CodeGenFunction &CGF, OpenMPProcBindClauseKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
llvm::Value * NewTaskNewTaskTTy
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
LValueBaseInfo getBaseInfo() const
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
QualType getPointeeType() const
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param Data Additional data for task generation like final list of privates etc *TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item)...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
CharUnits getAlignment() const
Return the alignment of this pointer.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc)
Call the appropriate runtime routine to notify that we finished all the work with current loop...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
OpenMPLocationFlags
Values for bit flags used in the ident_t to describe the fields.
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
OpenMPScheduleClauseModifier M1
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, unsigned Order)
Initialize target region entry.
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
llvm::Function * createOffloadingBinaryDescriptorRegistration()
Creates and registers offloading binary descriptor for the current compilation unit.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::IntegerType * IntPtrTy
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Value *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, const Expr *Device, ArrayRef< llvm::Value * > CapturedVars)
Emit the target offloading code associated with D.
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
bool empty() const
Return true if a there are no entries defined.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
detail::InMemoryDirectory::const_iterator E
void EmitAggregateCopy(Address DestPtr, Address SrcPtr, QualType EltTy, bool isVolatile=false, bool isAssignment=false)
EmitAggregateCopy - Emit an aggregate copy.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...
Not really used in Fortran any more.
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
const RecordType * getAsStructureType() const
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
const VariableArrayType * getAsVariableArrayType(QualType T) const
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
static CharUnits getIdentSize(CodeGenModule &CGM)
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
OffloadEntriesInfoManagerTy OffloadEntriesInfoManager
API for captured statement code generation.
llvm::PointerType * getType() const
Return the type of the pointer value.
static bool classof(const OMPClause *T)
const T * getAs() const
Member-template getAs<specific type>'.
QualType getCanonicalType() const
This file defines OpenMP AST classes for executable directives and clauses.
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
StructBuilder beginStruct(llvm::StructType *structTy=nullptr)
Address CreateConstArrayGEP(Address Addr, uint64_t Index, CharUnits EltSize, const llvm::Twine &Name="")
Given addr = [n x T]* ...
CleanupTy(PrePostActionTy *Action)
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc, unsigned &DeviceID, unsigned &FileID, unsigned &LineNum)
Obtain information that uniquely identifies a target entry.
llvm::PointerType * Int8PtrTy
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
LValue EmitLoadOfReferenceLValue(Address Ref, const ReferenceType *RefTy)
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Base for LValueReferenceType and RValueReferenceType.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
StringRef getMangledName(GlobalDecl GD)
QualType withRestrict() const
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void addDecl(Decl *D)
Add the declaration D into this context.
FieldDecl * LambdaThisCaptureField
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Constant * createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned...
QualType getPointeeType() const
SourceManager & getSourceManager()
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Lower bound for default (unordered) versions.
static std::string generateUniqueName(StringRef Prefix, SourceLocation Loc, unsigned N)
Generates unique name for artificial threadprivate variables.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Value *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
llvm::Function * CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false)
llvm::Value * BasePointersArray
The array of base pointer passed to the runtime library.
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
OpenMPOffloadingReservedDeviceIDs
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
llvm::Type * ConvertType(QualType T)
Privates[]
Gets the list of initial values for linear variables.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo=LValueBaseInfo(AlignmentSource::Type))
static const Stmt * ignoreCompoundStmts(const Stmt *Body)
discard all CompoundStmts intervening between two constructs
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
void popTerminate()
Pops a terminate handler off the stack.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
std::pair< llvm::Value *, QualType > getVLASize(const VariableArrayType *vla)
getVLASize - Returns an LLVM value that corresponds to the size, in non-variably-sized elements...
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
A reference to a declared variable, function, enum, etc.
static RValue get(llvm::Value *V)
const llvm::Triple & getTriple() const
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, ArrayType::ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type...
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
ParamKindTy
Kind of parameter in a function with 'declare simd' directive.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D)
Emit initialization for doacross loop nesting support.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
static int array_pod_sort_comparator(const PrivateDataTy *P1, const PrivateDataTy *P2)
An l-value expression is a reference to an object with independent storage.
static RValue getAggregate(Address addr, bool isVolatile=false)
SourceLocation getLocation() const
LValue - This represents an lvalue references.
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Information for lazily generating a cleanup.
llvm::Constant * createRuntimeFunction(unsigned Function)
Returns specified OpenMP runtime function.
SourceLocation getLocEnd() const
Returns ending location of directive.
bool isNull() const
Return true if this QualType doesn't point to a type yet.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
void operator()(CodeGenFunction &CGF) const
SourceLocation getLocStart() const LLVM_READONLY
Address CreateMemTemp(QualType T, const Twine &Name="tmp", bool CastToDefaultAddrSpace=true)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
A class which abstracts out some details necessary for making a call.
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
Attr - This represents one attribute.
SmallVector< const Expr *, 4 > FirstprivateInits
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object...
llvm::Value * SizesArray
The array of sizes passed to the runtime library.
Expr * getStrideVariable() const
bool Privatize()
Privatizes local variables previously registered as private.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
bool isPointerType() const
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.