22 #include "llvm/ADT/ArrayRef.h" 23 #include "llvm/Bitcode/BitcodeReader.h" 24 #include "llvm/IR/DerivedTypes.h" 25 #include "llvm/IR/GlobalValue.h" 26 #include "llvm/IR/Value.h" 27 #include "llvm/Support/Format.h" 28 #include "llvm/Support/raw_ostream.h" 31 using namespace clang;
32 using namespace CodeGen;
39 enum CGOpenMPRegionKind {
42 ParallelOutlinedRegion,
53 const CGOpenMPRegionKind RegionKind,
56 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
57 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
59 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
62 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
63 Kind(Kind), HasCancel(HasCancel) {}
67 virtual const VarDecl *getThreadIDVariable()
const = 0;
78 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
82 bool hasCancel()
const {
return HasCancel; }
84 static bool classof(
const CGCapturedStmtInfo *Info) {
88 ~CGOpenMPRegionInfo()
override =
default;
91 CGOpenMPRegionKind RegionKind;
98 class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
103 StringRef HelperName)
104 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
106 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
107 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
112 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
115 StringRef getHelperName()
const override {
return HelperName; }
117 static bool classof(
const CGCapturedStmtInfo *Info) {
119 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
120 ParallelOutlinedRegion;
127 StringRef HelperName;
131 class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
137 llvm::SwitchInst *UntiedSwitch =
nullptr;
140 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
142 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
152 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
156 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
158 emitUntiedSwitch(CGF);
173 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
179 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
185 const UntiedTaskActionTy &Action)
186 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
187 ThreadIDVar(ThreadIDVar), Action(Action) {
188 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
193 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
199 StringRef getHelperName()
const override {
return ".omp_outlined."; }
202 Action.emitUntiedSwitch(CGF);
205 static bool classof(
const CGCapturedStmtInfo *Info) {
207 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
216 const UntiedTaskActionTy &Action;
221 class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
226 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
228 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
233 return OuterRegionInfo->getContextValue();
234 llvm_unreachable(
"No context value for inlined OpenMP region");
238 if (OuterRegionInfo) {
239 OuterRegionInfo->setContextValue(V);
242 llvm_unreachable(
"No context value for inlined OpenMP region");
248 return OuterRegionInfo->lookup(VD);
254 FieldDecl *getThisFieldDecl()
const override {
256 return OuterRegionInfo->getThisFieldDecl();
262 const VarDecl *getThreadIDVariable()
const override {
264 return OuterRegionInfo->getThreadIDVariable();
271 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
272 llvm_unreachable(
"No LValue for inlined OpenMP construct");
276 StringRef getHelperName()
const override {
277 if (
auto *OuterRegionInfo = getOldCSI())
278 return OuterRegionInfo->getHelperName();
279 llvm_unreachable(
"No helper name for inlined OpenMP construct");
284 OuterRegionInfo->emitUntiedSwitch(CGF);
289 static bool classof(
const CGCapturedStmtInfo *Info) {
291 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
294 ~CGOpenMPInlinedRegionInfo()
override =
default;
299 CGOpenMPRegionInfo *OuterRegionInfo;
307 class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
311 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
313 HelperName(HelperName) {}
317 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
320 StringRef getHelperName()
const override {
return HelperName; }
322 static bool classof(
const CGCapturedStmtInfo *Info) {
324 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
328 StringRef HelperName;
332 llvm_unreachable(
"No codegen for expressions");
336 class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
339 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
346 for (
const auto &C : CS.
captures()) {
347 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
350 const VarDecl *VD =
C.getCapturedVar();
358 PrivScope.addPrivate(
361 (void)PrivScope.Privatize();
366 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
373 llvm_unreachable(
"No body for expressions");
378 const VarDecl *getThreadIDVariable()
const override {
379 llvm_unreachable(
"No thread id for expressions");
383 StringRef getHelperName()
const override {
384 llvm_unreachable(
"No helper name for expressions");
387 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
395 class InlinedOpenMPRegionRAII {
397 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
398 FieldDecl *LambdaThisCaptureField =
nullptr;
419 ~InlinedOpenMPRegionRAII() {
436 OMP_IDENT_IMD = 0x01,
438 OMP_IDENT_KMPC = 0x02,
440 OMP_ATOMIC_REDUCE = 0x10,
442 OMP_IDENT_BARRIER_EXPL = 0x20,
444 OMP_IDENT_BARRIER_IMPL = 0x40,
446 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
448 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
450 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
452 OMP_IDENT_WORK_LOOP = 0x200,
454 OMP_IDENT_WORK_SECTIONS = 0x400,
456 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
457 LLVM_MARK_AS_BITMASK_ENUM(OMP_IDENT_WORK_DISTRIBUTE)
465 OMP_REQ_UNDEFINED = 0x000,
467 OMP_REQ_NONE = 0x001,
469 OMP_REQ_REVERSE_OFFLOAD = 0x002,
471 OMP_REQ_UNIFIED_ADDRESS = 0x004,
473 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
475 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
476 LLVM_MARK_AS_BITMASK_ENUM(OMP_REQ_DYNAMIC_ALLOCATORS)
482 OMP_DEVICEID_UNDEF = -1,
777 Callback(CodeGen, CGF, *PrePostAction);
780 Callback(CodeGen, CGF, Action);
788 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
789 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
790 if (
const auto *DRE =
791 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
792 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
803 std::pair<llvm::Function *, llvm::Function *> Reduction =
805 const auto *CE = cast<CallExpr>(InitOp);
806 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
810 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
812 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
814 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
815 [=]() {
return Private; });
816 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
817 [=]() {
return Original; });
818 (void)PrivateScope.Privatize();
825 auto *GV =
new llvm::GlobalVariable(
827 llvm::GlobalValue::PrivateLinkage, Init, Name);
873 SrcBegin = SrcAddr.getPointer();
881 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
882 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
885 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
890 llvm::PHINode *SrcElementPHI =
nullptr;
893 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
894 "omp.arraycpy.srcElementPast");
895 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
898 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
900 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
901 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
902 DestElementPHI->addIncoming(DestBegin, EntryBB);
910 if (EmitDeclareReductionInit) {
912 SrcElementCurrent, ElementTy);
921 SrcElementPHI, 1,
"omp.arraycpy.dest.element");
922 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
927 DestElementPHI, 1,
"omp.arraycpy.dest.element");
930 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
931 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
932 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
944 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
949 void ReductionCodeGen::emitAggregateInitialization(
955 const auto *PrivateVD =
956 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
957 bool EmitDeclareReductionInit =
960 EmitDeclareReductionInit,
961 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
962 : PrivateVD->getInit(),
969 ClausesData.reserve(Shareds.size());
970 SharedAddresses.reserve(Shareds.size());
971 Sizes.reserve(Shareds.size());
972 BaseDecls.reserve(Shareds.size());
973 auto IPriv = Privates.begin();
974 auto IRed = ReductionOps.begin();
975 for (
const Expr *Ref : Shareds) {
976 ClausesData.emplace_back(Ref, *IPriv, *IRed);
977 std::advance(IPriv, 1);
978 std::advance(IRed, 1);
983 assert(SharedAddresses.size() == N &&
984 "Number of generated lvalues must be exactly N.");
985 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
986 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
987 SharedAddresses.emplace_back(First, Second);
991 const auto *PrivateVD =
992 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
993 QualType PrivateType = PrivateVD->getType();
994 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
995 if (!PrivateType->isVariablyModifiedType()) {
998 SharedAddresses[N].first.getType().getNonReferenceType()),
1005 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
1007 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1008 if (AsArraySection) {
1009 Size = CGF.
Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
1010 SharedAddresses[N].first.getPointer());
1011 Size = CGF.
Builder.CreateNUWAdd(
1012 Size, llvm::ConstantInt::get(Size->getType(), 1));
1013 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
1016 SharedAddresses[N].first.getType().getNonReferenceType());
1017 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1019 Sizes.emplace_back(SizeInChars, Size);
1022 cast<OpaqueValueExpr>(
1030 const auto *PrivateVD =
1031 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1032 QualType PrivateType = PrivateVD->getType();
1033 if (!PrivateType->isVariablyModifiedType()) {
1034 assert(!Size && !Sizes[N].second &&
1035 "Size should be nullptr for non-variably modified reduction " 1041 cast<OpaqueValueExpr>(
1050 assert(SharedAddresses.size() > N &&
"No variable was generated");
1051 const auto *PrivateVD =
1052 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1055 QualType PrivateType = PrivateVD->getType();
1058 QualType SharedType = SharedAddresses[N].first.getType();
1062 SharedType, SharedAddresses[N].first.getBaseInfo(),
1065 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1066 }
else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1070 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1073 PrivateVD->
getType().getQualifiers(),
1079 const auto *PrivateVD =
1080 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1081 QualType PrivateType = PrivateVD->getType();
1088 const auto *PrivateVD =
1089 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1090 QualType PrivateType = PrivateVD->getType();
1092 if (needCleanups(N)) {
1095 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
1144 return Address(Addr, BaseLVAlignment);
1148 const VarDecl *OrigVD =
nullptr;
1149 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1150 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
1151 while (
const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1153 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1155 DE = cast<DeclRefExpr>(Base);
1156 OrigVD = cast<VarDecl>(DE->
getDecl());
1157 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1158 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
1159 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1161 DE = cast<DeclRefExpr>(Base);
1162 OrigVD = cast<VarDecl>(DE->
getDecl());
1171 BaseDecls.emplace_back(OrigVD);
1174 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1175 OriginalBaseLValue);
1177 BaseLValue.
getPointer(), SharedAddresses[N].first.getPointer());
1181 SharedAddresses[N].first.getAddress().getType());
1184 SharedAddresses[N].first.getType(),
1188 BaseDecls.emplace_back(
1189 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1218 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1221 getThreadIDVariable()->
getType(),
1237 StringRef Separator)
1238 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1257 KmpCriticalNameTy = llvm::ArrayType::get(CGM.
Int32Ty, 8);
1263 InternalVars.clear();
1266 if (!Data.getValue().pointsToAliveValue())
1268 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1271 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1273 GV->eraseFromParent();
1279 llvm::raw_svector_ostream OS(Buffer);
1281 for (StringRef Part : Parts) {
1288 static llvm::Function *
1290 const Expr *CombinerInitializer,
const VarDecl *In,
1291 const VarDecl *Out,
bool IsCombiner) {
1300 Args.push_back(&OmpOutParm);
1301 Args.push_back(&OmpInParm);
1306 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1311 Fn->removeFnAttr(llvm::Attribute::NoInline);
1312 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1313 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1322 Scope.
addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1327 Scope.
addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1332 if (!IsCombiner && Out->
hasInit() &&
1338 if (CombinerInitializer)
1347 if (UDRMap.count(D) > 0)
1351 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1352 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1360 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1361 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1364 UDRMap.try_emplace(D, Combiner, Initializer);
1366 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->
CurFn);
1367 Decls.second.push_back(D);
1371 std::pair<llvm::Function *, llvm::Function *>
1373 auto I = UDRMap.find(D);
1374 if (I != UDRMap.end())
1377 return UDRMap.lookup(D);
1385 "thread id variable must be of type kmp_int32 *");
1387 bool HasCancel =
false;
1388 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1389 HasCancel = OPD->hasCancel();
1390 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1391 HasCancel = OPSD->hasCancel();
1392 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1393 HasCancel = OPFD->hasCancel();
1394 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1395 HasCancel = OPFD->hasCancel();
1396 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1397 HasCancel = OPFD->hasCancel();
1398 else if (
const auto *OPFD =
1399 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1400 HasCancel = OPFD->hasCancel();
1401 else if (
const auto *OPFD =
1402 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1403 HasCancel = OPFD->hasCancel();
1404 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1405 HasCancel, OutlinedHelperName);
1407 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1430 bool Tied,
unsigned &NumberOfParts) {
1437 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1442 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1446 "thread id variable must be of type kmp_int32 for tasks");
1453 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1455 TD ? TD->hasCancel() :
false, Action);
1457 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1459 NumberOfParts = Action.getNumberOfParts();
1465 ArrayRef<llvm::Constant *> Data) {
1467 unsigned PrevIdx = 0;
1469 auto DI = Data.begin();
1473 for (
unsigned I = PrevIdx; I < Idx; ++I)
1474 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1481 template <
class... As>
1482 static llvm::GlobalVariable *
1484 ArrayRef<llvm::Constant *> Data,
const Twine &Name,
1493 std::forward<As>(Args)...);
1496 template <
typename T>
1499 ArrayRef<llvm::Constant *> Data,
1508 Address CGOpenMPRuntime::getOrCreateDefaultLocation(
unsigned Flags) {
1511 FlagsTy FlagsKey(Flags, Reserved2Flags);
1512 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1514 if (!DefaultOpenMPPSource) {
1519 DefaultOpenMPPSource =
1521 DefaultOpenMPPSource =
1522 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource,
CGM.
Int8PtrTy);
1525 llvm::Constant *Data[] = {
1526 llvm::ConstantInt::getNullValue(
CGM.
Int32Ty),
1528 llvm::ConstantInt::get(
CGM.
Int32Ty, Reserved2Flags),
1529 llvm::ConstantInt::getNullValue(
CGM.
Int32Ty), DefaultOpenMPPSource};
1530 llvm::GlobalValue *DefaultOpenMPLocation =
1532 llvm::GlobalValue::PrivateLinkage);
1533 DefaultOpenMPLocation->setUnnamedAddr(
1534 llvm::GlobalValue::UnnamedAddr::Global);
1536 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1542 bool AtCurrentPoint) {
1543 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1544 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1547 if (AtCurrentPoint) {
1548 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1551 Elem.second.ServiceInsertPt =
1552 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1558 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1559 if (Elem.second.ServiceInsertPt) {
1560 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1561 Elem.second.ServiceInsertPt =
nullptr;
1562 Ptr->eraseFromParent();
1569 Flags |= OMP_IDENT_KMPC;
1573 return getOrCreateDefaultLocation(Flags).
getPointer();
1575 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1579 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
1580 if (I != OpenMPLocThreadIDMap.end())
1581 LocValue =
Address(I->second.DebugLoc, Align);
1588 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1592 if (!Elem.second.ServiceInsertPt)
1594 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1595 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1602 auto Fields = cast<RecordDecl>(IdentQTy->
getAsTagDecl())->field_begin();
1607 if (OMPDebugLoc ==
nullptr) {
1609 llvm::raw_svector_ostream OS2(Buffer2);
1613 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1614 OS2 << FD->getQualifiedNameAsString();
1616 OMPDebugLoc = CGF.
Builder.CreateGlobalStringPtr(OS2.str());
1629 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1634 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
1635 if (I != OpenMPLocThreadIDMap.end()) {
1636 ThreadID = I->second.ThreadID;
1637 if (ThreadID !=
nullptr)
1644 if (
auto *OMPRegionInfo =
1646 if (OMPRegionInfo->getThreadIDVariable()) {
1648 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1653 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1654 Elem.second.ThreadID = ThreadID;
1665 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1666 if (!Elem.second.ServiceInsertPt)
1668 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1669 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1670 llvm::CallInst *Call = CGF.
Builder.CreateCall(
1674 Elem.second.ThreadID = Call;
1679 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1680 if (OpenMPLocThreadIDMap.count(CGF.
CurFn)) {
1682 OpenMPLocThreadIDMap.erase(CGF.
CurFn);
1684 if (FunctionUDRMap.count(CGF.
CurFn) > 0) {
1685 for(
auto *D : FunctionUDRMap[CGF.
CurFn])
1687 FunctionUDRMap.erase(CGF.
CurFn);
1692 return IdentTy->getPointerTo();
1696 if (!Kmpc_MicroTy) {
1700 Kmpc_MicroTy = llvm::FunctionType::get(
CGM.
VoidTy, MicroParams,
true);
1702 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1706 llvm::FunctionCallee RTLFn =
nullptr;
1707 switch (static_cast<OpenMPRTLFunction>(Function)) {
1714 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
true);
1716 if (
auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1717 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1718 llvm::LLVMContext &Ctx = F->getContext();
1719 llvm::MDBuilder MDB(Ctx);
1726 llvm::LLVMContext::MD_callback,
1727 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1738 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1749 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
1758 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1760 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1768 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1771 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1781 false)->getPointerTo();
1784 auto *KmpcCopyCtorTy =
1785 llvm::FunctionType::get(
CGM.
VoidPtrTy, KmpcCopyCtorTyArgs,
1793 KmpcCopyCtorTy, KmpcDtorTy};
1794 auto *FnTy = llvm::FunctionType::get(
CGM.
VoidTy, FnTyArgs,
1804 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1806 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1815 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1823 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1831 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1841 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1850 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1859 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1867 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1875 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1883 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1892 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1900 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1908 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1916 assert(KmpRoutineEntryPtrTy !=
nullptr &&
1917 "Type kmp_routine_entry_t must be created.");
1922 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
1930 assert(KmpRoutineEntryPtrTy !=
nullptr &&
1931 "Type kmp_routine_entry_t must be created.");
1937 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
1947 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1957 llvm::FunctionType::get(
CGM.
VoidTy, CpyTypeParams,
false);
1962 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1971 auto *ReduceFnTy = llvm::FunctionType::get(
CGM.
VoidTy, ReduceTypeParams,
1976 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1978 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1988 auto *ReduceFnTy = llvm::FunctionType::get(
CGM.
VoidTy, ReduceTypeParams,
1993 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1995 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2004 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2006 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2015 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2017 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2028 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2039 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2041 "__kmpc_omp_task_complete_if0");
2048 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2056 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2064 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2072 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2080 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2089 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2101 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2114 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2123 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2132 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2142 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2152 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
true);
2154 if (
auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2155 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2156 llvm::LLVMContext &Ctx = F->getContext();
2157 llvm::MDBuilder MDB(Ctx);
2164 llvm::LLVMContext::MD_callback,
2165 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2188 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2200 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2208 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2218 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2228 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2237 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2247 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2249 FnTy,
"__kmpc_task_reduction_get_th_data");
2257 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2266 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2274 llvm::FunctionType *FnTy =
2275 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2291 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2307 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2325 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2343 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2351 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2361 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2371 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2385 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2400 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2414 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2429 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2443 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2458 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2463 assert(RTLFn &&
"Unable to find OpenMP runtime function");
2467 llvm::FunctionCallee
2469 assert((IVSize == 32 || IVSize == 64) &&
2470 "IV size is not compatible with the omp runtime");
2471 StringRef Name = IVSize == 32 ? (IVSigned ?
"__kmpc_for_static_init_4" 2472 :
"__kmpc_for_static_init_4u")
2473 : (IVSigned ?
"__kmpc_for_static_init_8" 2474 :
"__kmpc_for_static_init_8u");
2476 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2489 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2493 llvm::FunctionCallee
2495 assert((IVSize == 32 || IVSize == 64) &&
2496 "IV size is not compatible with the omp runtime");
2499 ? (IVSigned ?
"__kmpc_dispatch_init_4" :
"__kmpc_dispatch_init_4u")
2500 : (IVSigned ?
"__kmpc_dispatch_init_8" :
"__kmpc_dispatch_init_8u");
2511 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2515 llvm::FunctionCallee
2517 assert((IVSize == 32 || IVSize == 64) &&
2518 "IV size is not compatible with the omp runtime");
2521 ? (IVSigned ?
"__kmpc_dispatch_fini_4" :
"__kmpc_dispatch_fini_4u")
2522 : (IVSigned ?
"__kmpc_dispatch_fini_8" :
"__kmpc_dispatch_fini_8u");
2528 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2532 llvm::FunctionCallee
2534 assert((IVSize == 32 || IVSize == 64) &&
2535 "IV size is not compatible with the omp runtime");
2538 ? (IVSigned ?
"__kmpc_dispatch_next_4" :
"__kmpc_dispatch_next_4u")
2539 : (IVSigned ?
"__kmpc_dispatch_next_8" :
"__kmpc_dispatch_next_8u");
2541 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2551 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2559 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2560 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2561 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2565 llvm::raw_svector_ostream OS(PtrName);
2574 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2591 std::string Suffix =
getName({
"cache",
""});
2627 Ctor, CopyCtor, Dtor};
2643 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
2653 Args.push_back(&Dst);
2658 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
2659 llvm::Function *Fn =
2686 Args.push_back(&Dst);
2691 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
2692 llvm::Function *Fn =
2713 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
2719 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2720 if (Ctor ==
nullptr) {
2724 Ctor = llvm::Constant::getNullValue(CtorTy);
2726 if (Dtor ==
nullptr) {
2730 Dtor = llvm::Constant::getNullValue(DtorTy);
2733 auto *InitFunctionTy =
2734 llvm::FunctionType::get(
CGM.
VoidTy,
false);
2735 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
2745 return InitFunction;
2756 unsigned &DeviceID,
unsigned &
FileID,
2757 unsigned &LineNum) {
2763 assert(Loc.
isValid() &&
"Source location is expected to be always valid.");
2766 assert(PLoc.isValid() &&
"Source location is expected to be always valid.");
2768 llvm::sys::fs::UniqueID
ID;
2769 if (
auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(),
ID))
2771 << PLoc.getFilename() << EC.message();
2773 DeviceID = ID.getDevice();
2774 FileID = ID.getFile();
2775 LineNum = PLoc.getLine();
2779 llvm::GlobalVariable *Addr,
2782 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2783 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
2784 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2803 llvm::raw_svector_ostream OS(Buffer);
2804 OS <<
"__omp_offloading_" << llvm::format(
"_%x", DeviceID)
2805 << llvm::format(
"_%x_", FileID) << VD->
getName() <<
"_l" <<
Line;
2810 llvm::Constant *Ctor;
2820 FTy, Twine(Buffer,
"_ctor"), FI, Loc);
2831 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
2834 Ctor =
new llvm::GlobalVariable(
2836 llvm::GlobalValue::PrivateLinkage,
2837 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_ctor"));
2844 DeviceID, FileID, Twine(Buffer,
"_ctor").toStringRef(Out), Line, Ctor,
2848 llvm::Constant *Dtor;
2858 FTy, Twine(Buffer,
"_dtor"), FI, Loc);
2870 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
2873 Dtor =
new llvm::GlobalVariable(
2875 llvm::GlobalValue::PrivateLinkage,
2876 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_dtor"));
2882 DeviceID, FileID, Twine(Buffer,
"_dtor").toStringRef(Out), Line, Dtor,
2891 std::string Suffix =
getName({
"artificial",
""});
2892 std::string CacheSuffix =
getName({
"cache",
""});
2908 VarLVType->getPointerTo(0)),
2952 llvm::Function *OutlinedFn,
2954 const Expr *IfCond) {
2958 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](
CodeGenFunction &CGF,
2964 CGF.Builder.getInt32(CapturedVars.size()),
2965 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2967 RealArgs.append(std::begin(Args), std::end(Args));
2968 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2970 llvm::FunctionCallee RTLFn =
2972 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2974 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](
CodeGenFunction &CGF,
2981 CGF.EmitRuntimeCall(
2985 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2987 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32( 0));
2990 OutlinedFnArgs.push_back(ZeroAddr.
getPointer());
2991 OutlinedFnArgs.push_back(ZeroAddr.
getPointer());
2992 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2993 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2997 CGF.EmitRuntimeCall(
3017 if (
auto *OMPRegionInfo =
3019 if (OMPRegionInfo->getThreadIDVariable())
3020 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
3029 return ThreadIDTemp;
3033 llvm::Type *Ty,
const llvm::Twine &Name,
unsigned AddressSpace) {
3035 llvm::raw_svector_ostream Out(Buffer);
3037 StringRef RuntimeName = Out.str();
3038 auto &Elem = *InternalVars.try_emplace(RuntimeName,
nullptr).first;
3040 assert(Elem.second->getType()->getPointerElementType() == Ty &&
3041 "OMP internal variable has different type than requested");
3042 return &*Elem.second;
3045 return Elem.second =
new llvm::GlobalVariable(
3047 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3048 Elem.first(),
nullptr,
3049 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3053 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
3054 std::string Name =
getName({Prefix,
"var"});
3061 llvm::FunctionCallee EnterCallee;
3063 llvm::FunctionCallee ExitCallee;
3066 llvm::BasicBlock *ContBlock =
nullptr;
3069 CommonActionTy(llvm::FunctionCallee EnterCallee,
3071 llvm::FunctionCallee ExitCallee,
3073 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3082 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3098 StringRef CriticalName,
3112 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
3115 CommonActionTy Action(
3149 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
3151 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
3152 Region->emitUntiedSwitch(CGF);
3175 unsigned Index,
const VarDecl *Var) {
3188 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3189 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3198 Args.push_back(&LHSArg);
3199 Args.push_back(&RHSArg);
3208 Fn->setDoesNotRecurse();
3223 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3224 const auto *DestVar =
3225 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3228 const auto *SrcVar =
3229 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3232 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3234 CGF.
EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3243 ArrayRef<const Expr *> CopyprivateVars,
3244 ArrayRef<const Expr *> SrcExprs,
3245 ArrayRef<const Expr *> DstExprs,
3246 ArrayRef<const Expr *> AssignmentOps) {
3249 assert(CopyprivateVars.size() == SrcExprs.size() &&
3250 CopyprivateVars.size() == DstExprs.size() &&
3251 CopyprivateVars.size() == AssignmentOps.size());
3263 if (!CopyprivateVars.empty()) {
3266 C.getIntTypeForBitwidth(32, 1);
3267 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
3285 llvm::APInt ArraySize(32, CopyprivateVars.size());
3291 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
3292 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3303 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3344 if (Kind == OMPD_for)
3345 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3346 else if (Kind == OMPD_sections)
3347 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3348 else if (Kind == OMPD_single)
3349 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3350 else if (Kind == OMPD_barrier)
3351 Flags = OMP_IDENT_BARRIER_EXPL;
3353 Flags = OMP_IDENT_BARRIER_IMPL;
3365 ScheduleKind = OMPC_SCHEDULE_static;
3367 llvm::APInt ChunkSize(32, 1);
3377 bool ForceSimpleCall) {
3387 if (
auto *OMPRegionInfo =
3389 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3399 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3415 bool Chunked,
bool Ordered) {
3416 switch (ScheduleKind) {
3417 case OMPC_SCHEDULE_static:
3420 case OMPC_SCHEDULE_dynamic:
3422 case OMPC_SCHEDULE_guided:
3424 case OMPC_SCHEDULE_runtime:
3426 case OMPC_SCHEDULE_auto:
3429 assert(!Chunked &&
"chunk was specified but schedule kind not known");
3432 llvm_unreachable(
"Unexpected runtime schedule");
3443 bool Chunked)
const {
3456 bool Chunked)
const {
3480 case OMPC_SCHEDULE_MODIFIER_monotonic:
3483 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3486 case OMPC_SCHEDULE_MODIFIER_simd:
3495 case OMPC_SCHEDULE_MODIFIER_monotonic:
3498 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3501 case OMPC_SCHEDULE_MODIFIER_simd:
3519 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
3531 : CGF.
Builder.getIntN(IVSize, 1);
3535 Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
3538 CGF.
Builder.getIntN(IVSize, 1),
3548 const CGOpenMPRuntime::StaticRTInput &Values) {
3552 assert(!Values.Ordered);
3565 if (Chunk ==
nullptr) {
3568 "expected static non-chunked schedule");
3570 Chunk = CGF.
Builder.getIntN(Values.IVSize, 1);
3576 "expected static chunked schedule");
3583 Values.IL.getPointer(),
3584 Values.LB.getPointer(),
3585 Values.UB.getPointer(),
3586 Values.ST.getPointer(),
3587 CGF.
Builder.getIntN(Values.IVSize, 1),
3601 "Expected loop-based or sections-based directive.");
3604 ? OMP_IDENT_WORK_LOOP
3605 : OMP_IDENT_WORK_SECTIONS);
3607 llvm::FunctionCallee StaticInitFunction =
3610 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
3616 const CGOpenMPRuntime::StaticRTInput &Values) {
3622 llvm::FunctionCallee StaticInitFunction =
3638 ? OMP_IDENT_WORK_DISTRIBUTE
3640 ? OMP_IDENT_WORK_LOOP
3641 : OMP_IDENT_WORK_SECTIONS),
3711 case OMPC_PROC_BIND_master:
3712 RuntimeProcBind = ProcBindMaster;
3714 case OMPC_PROC_BIND_close:
3715 RuntimeProcBind = ProcBindClose;
3717 case OMPC_PROC_BIND_spread:
3718 RuntimeProcBind = ProcBindSpread;
3721 llvm_unreachable(
"Unsupported proc_bind value.");
3726 llvm::ConstantInt::get(
CGM.
IntTy, RuntimeProcBind,
true)};
3765 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty()
const {
3766 return OffloadEntriesTargetRegion.empty() &&
3767 OffloadEntriesDeviceGlobalVar.empty();
3771 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3772 initializeTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
3773 StringRef ParentName,
unsigned LineNum,
3775 assert(
CGM.
getLangOpts().OpenMPIsDevice &&
"Initialization of entries is " 3776 "only required for the device " 3777 "code generation.");
3778 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3779 OffloadEntryInfoTargetRegion(Order,
nullptr,
nullptr,
3780 OMPTargetRegionEntryTargetRegion);
3781 ++OffloadingEntriesNum;
3784 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3785 registerTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
3786 StringRef ParentName,
unsigned LineNum,
3787 llvm::Constant *Addr, llvm::Constant *
ID,
3788 OMPTargetRegionEntryKind Flags) {
3792 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3795 "Unable to find target region on line '%0' in the device code.");
3800 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3801 assert(Entry.isValid() &&
"Entry not initialized!");
3802 Entry.setAddress(Addr);
3804 Entry.setFlags(Flags);
3806 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3807 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3808 ++OffloadingEntriesNum;
3812 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3813 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
3814 unsigned LineNum)
const {
3815 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3816 if (PerDevice == OffloadEntriesTargetRegion.end())
3818 auto PerFile = PerDevice->second.find(FileID);
3819 if (PerFile == PerDevice->second.end())
3821 auto PerParentName = PerFile->second.find(ParentName);
3822 if (PerParentName == PerFile->second.end())
3824 auto PerLine = PerParentName->second.find(LineNum);
3825 if (PerLine == PerParentName->second.end())
3828 if (PerLine->second.getAddress() || PerLine->second.getID())
3833 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3834 const OffloadTargetRegionEntryInfoActTy &Action) {
3836 for (
const auto &D : OffloadEntriesTargetRegion)
3837 for (
const auto &F : D.second)
3838 for (
const auto &
P : F.second)
3839 for (
const auto &L :
P.second)
3840 Action(D.first, F.first,
P.first(), L.first, L.second);
3843 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3844 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3845 OMPTargetGlobalVarEntryKind Flags,
3847 assert(
CGM.
getLangOpts().OpenMPIsDevice &&
"Initialization of entries is " 3848 "only required for the device " 3849 "code generation.");
3850 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3851 ++OffloadingEntriesNum;
3854 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3855 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3857 OMPTargetGlobalVarEntryKind Flags,
3858 llvm::GlobalValue::LinkageTypes
Linkage) {
3860 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3861 assert(Entry.isValid() && Entry.getFlags() == Flags &&
3862 "Entry not initialized!");
3863 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
3864 "Resetting with the new address.");
3865 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3866 if (Entry.getVarSize().isZero()) {
3867 Entry.setVarSize(VarSize);
3868 Entry.setLinkage(Linkage);
3872 Entry.setVarSize(VarSize);
3873 Entry.setLinkage(Linkage);
3874 Entry.setAddress(Addr);
3876 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3877 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3878 assert(Entry.isValid() && Entry.getFlags() == Flags &&
3879 "Entry not initialized!");
3880 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
3881 "Resetting with the new address.");
3882 if (Entry.getVarSize().isZero()) {
3883 Entry.setVarSize(VarSize);
3884 Entry.setLinkage(Linkage);
3888 OffloadEntriesDeviceGlobalVar.try_emplace(
3889 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3890 ++OffloadingEntriesNum;
3894 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3895 actOnDeviceGlobalVarEntriesInfo(
3896 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3898 for (
const auto &E : OffloadEntriesDeviceGlobalVar)
3899 Action(E.getKey(), E.getValue());
3917 assert(!Devices.empty() &&
"No OpenMP offloading devices??");
3923 std::string EntriesBeginName =
getName({
"omp_offloading",
"entries_begin"});
3924 auto *HostEntriesBegin =
new llvm::GlobalVariable(
3925 M, OffloadEntryTy,
true,
3928 std::string EntriesEndName =
getName({
"omp_offloading",
"entries_end"});
3929 auto *HostEntriesEnd =
3930 new llvm::GlobalVariable(M, OffloadEntryTy,
true,
3932 nullptr, EntriesEndName);
3935 auto *DeviceImageTy = cast<llvm::StructType>(
3939 DeviceImagesBuilder.
beginArray(DeviceImageTy);
3941 for (
const llvm::Triple &Device : Devices) {
3942 StringRef T = Device.getTriple();
3943 std::string BeginName =
getName({
"omp_offloading",
"img_start",
""});
3944 auto *ImgBegin =
new llvm::GlobalVariable(
3946 llvm::GlobalValue::ExternalWeakLinkage,
3947 nullptr, Twine(BeginName).concat(T));
3948 std::string EndName =
getName({
"omp_offloading",
"img_end",
""});
3949 auto *ImgEnd =
new llvm::GlobalVariable(
3951 llvm::GlobalValue::ExternalWeakLinkage,
3952 nullptr, Twine(EndName).concat(T));
3954 llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
3957 DeviceImagesEntries);
3961 std::string ImagesName =
getName({
"omp_offloading",
"device_images"});
3962 llvm::GlobalVariable *DeviceImages =
3966 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3969 llvm::Constant *Index[] = {llvm::Constant::getNullValue(
CGM.
Int32Ty),
3973 llvm::Constant *Data[] = {
3974 llvm::ConstantInt::get(
CGM.
Int32Ty, Devices.size()),
3975 llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
3976 DeviceImages, Index),
3977 HostEntriesBegin, HostEntriesEnd};
3978 std::string Descriptor =
getName({
"omp_offloading",
"descriptor"});
3985 llvm::Function *UnRegFn;
3989 Args.push_back(&DummyPtr);
3998 std::string UnregName =
getName({
"omp_offloading",
"descriptor_unreg"});
4005 llvm::Function *RegFn;
4018 RegFnNameParts[0] =
"omp_offloading";
4019 RegFnNameParts[1] =
"descriptor_reg";
4020 llvm::transform(Devices, std::next(RegFnNameParts.begin(), 2),
4021 [](
const llvm::Triple &T) ->
const std::string& {
4022 return T.getTriple();
4024 llvm::sort(std::next(RegFnNameParts.begin(), 2), RegFnNameParts.end());
4025 std::string Descriptor =
getName(RegFnNameParts);
4042 llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
4043 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
4045 RegFn->setComdat(ComdatKey);
4046 UnRegFn->setComdat(ComdatKey);
4047 DeviceImages->setComdat(ComdatKey);
4048 Desc->setComdat(ComdatKey);
4054 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4055 llvm::GlobalValue::LinkageTypes Linkage) {
4056 StringRef Name = Addr->getName();
4058 llvm::LLVMContext &C = M.getContext();
4061 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4063 std::string StringName =
getName({
"omp_offloading",
"entry_name"});
4064 auto *Str =
new llvm::GlobalVariable(
4065 M, StrPtrInit->getType(),
true,
4067 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4069 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID,
CGM.
VoidPtrTy),
4071 llvm::ConstantInt::get(
CGM.
SizeTy, Size),
4074 std::string EntryName =
getName({
"omp_offloading",
"entry",
""});
4077 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4080 std::string Section =
getName({
"omp_offloading",
"entries"});
4081 Entry->setSection(Section);
4099 llvm::LLVMContext &C = M.getContext();
4106 auto &&GetMDInt = [
this](
unsigned V) {
4107 return llvm::ConstantAsMetadata::get(
4111 auto &&GetMDString = [&C](StringRef
V) {
return llvm::MDString::get(C, V); };
4114 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(
"omp_offload.info");
4117 auto &&TargetRegionMetadataEmitter =
4118 [&C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, &GetMDString](
4119 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
4132 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4133 GetMDInt(FileID), GetMDString(ParentName),
4134 GetMDInt(Line), GetMDInt(E.getOrder())};
4137 OrderedEntries[E.getOrder()] = &E;
4138 ParentFunctions[E.getOrder()] = ParentName;
4141 MD->addOperand(llvm::MDNode::get(C, Ops));
4145 TargetRegionMetadataEmitter);
4148 auto &&DeviceGlobalVarMetadataEmitter =
4149 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4150 MD](StringRef MangledName,
4160 llvm::Metadata *Ops[] = {
4161 GetMDInt(E.getKind()), GetMDString(MangledName),
4162 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4165 OrderedEntries[E.getOrder()] = &E;
4168 MD->addOperand(llvm::MDNode::get(C, Ops));
4172 DeviceGlobalVarMetadataEmitter);
4174 for (
const auto *E : OrderedEntries) {
4175 assert(E &&
"All ordered entries must exist!");
4176 if (
const auto *CE =
4177 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4179 if (!CE->getID() || !CE->getAddress()) {
4181 StringRef FnName = ParentFunctions[CE->getOrder()];
4186 "Offloading entry for target region is incorrect: either the " 4187 "address or the ID is invalid.");
4192 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4193 }
else if (
const auto *CE =
4195 OffloadEntryInfoDeviceGlobalVar>(E)) {
4204 if (!CE->getAddress()) {
4207 "Offloading entry for declare target variable is incorrect: the " 4208 "address is invalid.");
4213 if (CE->getVarSize().isZero())
4218 assert(((
CGM.
getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||
4220 "Declaret target link address is set.");
4223 if (!CE->getAddress()) {
4226 "Offloading entry for declare target variable is incorrect: the " 4227 "address is invalid.");
4234 CE->getVarSize().getQuantity(), Flags,
4237 llvm_unreachable(
"Unsupported entry kind.");
4255 if (
auto EC = Buf.getError()) {
4261 llvm::LLVMContext C;
4262 auto ME = expectedToErrorOrAndEmitErrors(
4263 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4265 if (
auto EC = ME.getError()) {
4273 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata(
"omp_offload.info");
4277 for (llvm::MDNode *MN : MD->operands()) {
4278 auto &&GetMDInt = [MN](
unsigned Idx) {
4279 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4280 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4283 auto &&GetMDString = [MN](
unsigned Idx) {
4284 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4285 return V->getString();
4288 switch (GetMDInt(0)) {
4290 llvm_unreachable(
"Unexpected metadata!");
4295 GetMDInt(1), GetMDInt(2),
4296 GetMDString(3), GetMDInt(4),
4303 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4312 if (!KmpRoutineEntryPtrTy) {
4346 RD->
addAttr(PackedAttr::CreateImplicit(C));
4404 struct PrivateHelpersTy {
4405 PrivateHelpersTy(
const VarDecl *Original,
const VarDecl *PrivateCopy,
4406 const VarDecl *PrivateElemInit)
4407 : Original(Original), PrivateCopy(PrivateCopy),
4408 PrivateElemInit(PrivateElemInit) {}
4411 const VarDecl *PrivateElemInit;
4413 typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
4418 if (!Privates.empty()) {
4425 for (
const auto &Pair : Privates) {
4426 const VarDecl *VD = Pair.second.Original;
4445 QualType KmpRoutineEntryPointerQTy) {
4490 ArrayRef<PrivateDataTy>
Privates) {
4516 static llvm::Function *
4519 QualType KmpTaskTWithPrivatesPtrQTy,
4521 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4530 Args.push_back(&GtidArg);
4531 Args.push_back(&TaskTypeArg);
4532 const auto &TaskEntryFnInfo =
4534 llvm::FunctionType *TaskEntryTy =
4540 TaskEntry->setDoesNotRecurse();
4555 const auto *KmpTaskTWithPrivatesQTyRD =
4556 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
4559 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
4560 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4562 llvm::Value *PartidParam = PartIdLVal.getPointer();
4564 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4570 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4572 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4577 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4580 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4587 std::end(CommonArgs));
4589 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4592 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4595 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4598 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4601 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4604 CallArgs.push_back(LBParam);
4605 CallArgs.push_back(UBParam);
4606 CallArgs.push_back(StParam);
4607 CallArgs.push_back(LIParam);
4608 CallArgs.push_back(RParam);
4610 CallArgs.push_back(SharedsParam);
4623 QualType KmpTaskTWithPrivatesPtrQTy,
4624 QualType KmpTaskTWithPrivatesQTy) {
4632 Args.push_back(&GtidArg);
4633 Args.push_back(&TaskTypeArg);
4634 const auto &DestructorFnInfo =
4636 llvm::FunctionType *DestructorFnTy =
4640 auto *DestructorFn =
4645 DestructorFn->setDoesNotRecurse();
4653 const auto *KmpTaskTWithPrivatesQTyRD =
4654 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
4655 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4657 for (
const auto *Field :
4658 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4660 Field->getType().isDestructedType()) {
4666 return DestructorFn;
4681 ArrayRef<const Expr *> PrivateVars,
4682 ArrayRef<const Expr *> FirstprivateVars,
4683 ArrayRef<const Expr *> LastprivateVars,
4685 ArrayRef<PrivateDataTy>
Privates) {
4689 C,
nullptr, Loc,
nullptr,
4692 Args.push_back(&TaskPrivatesArg);
4693 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4694 unsigned Counter = 1;
4695 for (
const Expr *E : PrivateVars) {
4697 C,
nullptr, Loc,
nullptr,
4702 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4703 PrivateVarsPos[VD] = Counter;
4706 for (
const Expr *E : FirstprivateVars) {
4708 C,
nullptr, Loc,
nullptr,
4713 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4714 PrivateVarsPos[VD] = Counter;
4717 for (
const Expr *E : LastprivateVars) {
4719 C,
nullptr, Loc,
nullptr,
4724 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4725 PrivateVarsPos[VD] = Counter;
4728 const auto &TaskPrivatesMapFnInfo =
4730 llvm::FunctionType *TaskPrivatesMapTy =
4738 TaskPrivatesMapFnInfo);
4740 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4741 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4742 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4746 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4752 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
4754 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
4756 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4765 return TaskPrivatesMap;
4775 ArrayRef<PrivateDataTy>
Privates,
bool ForDup) {
4777 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
4792 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
4798 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4799 for (
const PrivateDataTy &Pair : Privates) {
4800 const VarDecl *VD = Pair.second.PrivateCopy;
4802 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4805 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
4806 const VarDecl *OriginalVD = Pair.second.Original;
4812 if (IsTargetTask && !SharedField) {
4813 assert(isa<ImplicitParamDecl>(OriginalVD) &&
4816 ->getNumParams() == 0 &&
4817 isa<TranslationUnitDecl>(
4819 ->getDeclContext()) &&
4820 "Expected artificial target data variable.");
4840 [&CGF, Elem, Init, &CapturesInfo](
Address DestElement,
4845 Elem, [SrcElement]() ->
Address {
return SrcElement; });
4849 CGF, &CapturesInfo);
4850 CGF.EmitAnyExprToMem(Init, DestElement,
4851 Init->getType().getQualifiers(),
4875 ArrayRef<PrivateDataTy>
Privates) {
4876 bool InitRequired =
false;
4877 for (
const PrivateDataTy &Pair : Privates) {
4878 const VarDecl *VD = Pair.second.PrivateCopy;
4880 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4885 return InitRequired;
4902 QualType KmpTaskTWithPrivatesPtrQTy,
4906 ArrayRef<PrivateDataTy>
Privates,
bool WithLastIter) {
4910 KmpTaskTWithPrivatesPtrQTy,
4913 KmpTaskTWithPrivatesPtrQTy,
4917 Args.push_back(&DstArg);
4918 Args.push_back(&SrcArg);
4919 Args.push_back(&LastprivArg);
4920 const auto &TaskDupFnInfo =
4927 TaskDup->setDoesNotRecurse();
4937 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
4939 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
4947 assert(!Privates.empty());
4954 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
4962 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4963 SharedsTy, SharedsPtrTy, Data, Privates,
true);
4972 bool NeedsCleanup =
false;
4973 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin(), 1);
4974 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4975 for (
const FieldDecl *FD : PrivateRD->fields()) {
4976 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4980 return NeedsCleanup;
4983 CGOpenMPRuntime::TaskResultTy
4986 llvm::Function *TaskFunction,
QualType SharedsTy,
4993 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4994 Privates.emplace_back(
4996 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5003 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5004 Privates.emplace_back(
5007 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5008 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
5014 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5015 Privates.emplace_back(
5017 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5021 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5022 return L.first > R.first;
5038 "Expected taskloop, task or target directive");
5047 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5050 QualType KmpTaskTWithPrivatesPtrQTy =
5054 KmpTaskTWithPrivatesTy->getPointerTo();
5062 std::next(TaskFunction->arg_begin(), 3)->getType();
5063 if (!Privates.empty()) {
5064 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5069 TaskPrivatesMap, TaskPrivatesMapTy);
5071 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5072 cast<llvm::PointerType>(TaskPrivatesMapTy));
5078 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5090 DestructorsFlag = 0x8,
5093 unsigned Flags = Data.
Tied ? TiedFlag : 0;
5094 bool NeedsCleanup =
false;
5095 if (!Privates.empty()) {
5098 Flags = Flags | DestructorsFlag;
5101 Flags = Flags | PriorityFlag;
5103 Data.
Final.getPointer()
5105 CGF.
Builder.getInt32(FinalFlag),
5107 : CGF.
Builder.getInt32(Data.
Final.getInt() ? FinalFlag : 0);
5108 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
5111 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5113 TaskEntry, KmpRoutineEntryPtrTy)};
5117 const Expr *Device =
nullptr;
5119 Device = C->getDevice();
5126 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
5127 AllocArgs.push_back(DeviceID);
5136 NewTask, KmpTaskTWithPrivatesPtrTy);
5138 KmpTaskTWithPrivatesQTy);
5148 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5158 if (!Privates.empty()) {
5159 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5160 SharedsTy, SharedsPtrTy, Data, Privates,
5165 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5166 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5171 enum { Priority = 0, Destructors = 1 };
5173 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5175 (*FI)->getType()->getAsUnionType()->getDecl();
5178 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5179 KmpTaskTWithPrivatesQTy);
5182 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5184 DestructorFn, KmpRoutineEntryPtrTy),
5190 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5192 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5205 llvm::Function *TaskFunction,
5213 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5215 llvm::Function *TaskEntry = Result.
TaskEntry;
5222 unsigned NumDependencies = Data.
Dependences.size();
5223 if (NumDependencies) {
5225 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5226 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5249 for (
unsigned I = 0; I < NumDependencies; ++I) {
5254 if (
const auto *ASE =
5263 Size = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5272 Base, *std::next(KmpDependInfoRD->
field_begin(), BaseAddr));
5278 Base, *std::next(KmpDependInfoRD->
field_begin(), Len));
5281 RTLDependenceKindTy DepKind;
5283 case OMPC_DEPEND_in:
5287 case OMPC_DEPEND_out:
5288 case OMPC_DEPEND_inout:
5291 case OMPC_DEPEND_mutexinoutset:
5292 DepKind = DepMutexInOutSet;
5294 case OMPC_DEPEND_source:
5295 case OMPC_DEPEND_sink:
5297 llvm_unreachable(
"Unknown task dependence type");
5300 Base, *std::next(KmpDependInfoRD->
field_begin(), Flags));
5316 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5318 if (NumDependencies) {
5319 DepTaskArgs[0] = UpLoc;
5320 DepTaskArgs[1] = ThreadID;
5321 DepTaskArgs[2] = NewTask;
5322 DepTaskArgs[3] = CGF.
Builder.getInt32(NumDependencies);
5323 DepTaskArgs[4] = DependenciesArray.
getPointer();
5324 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
5325 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5327 auto &&ThenCodeGen = [
this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5331 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5332 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5333 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5335 if (NumDependencies) {
5336 CGF.EmitRuntimeCall(
5344 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5345 Region->emitUntiedSwitch(CGF);
5349 if (NumDependencies) {
5350 DepWaitTaskArgs[0] = UpLoc;
5351 DepWaitTaskArgs[1] = ThreadID;
5352 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5353 DepWaitTaskArgs[3] = DependenciesArray.
getPointer();
5354 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5355 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5357 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5358 NumDependencies, &DepWaitTaskArgs,
5366 if (NumDependencies)
5370 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5373 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5374 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5383 CommonActionTy Action(
5400 llvm::Function *TaskFunction,
5407 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5420 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
5454 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5463 llvm::ConstantInt::getSigned(
5465 llvm::ConstantInt::getSigned(
5467 ? Data.
Schedule.getInt() ? NumTasks : Grainsize
5472 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
5475 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
5492 const Expr *,
const Expr *)> &RedOpGen,
5493 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
5494 const Expr *UpExpr =
nullptr) {
5512 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
5513 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5516 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
5521 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
5522 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
5523 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5528 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
5529 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
5530 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5537 Scope.addPrivate(LHSVar, [=]() {
return LHSElementCurrent; });
5538 Scope.addPrivate(RHSVar, [=]() {
return RHSElementCurrent; });
5540 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5541 Scope.ForceCleanup();
5545 LHSElementPHI, 1,
"omp.arraycpy.dest.element");
5547 RHSElementPHI, 1,
"omp.arraycpy.src.element");
5550 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
5551 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
5552 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
5553 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
5563 const Expr *ReductionOp) {
5564 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5565 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5566 if (
const auto *DRE =
5567 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5568 if (
const auto *DRD =
5569 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5570 std::pair<llvm::Function *, llvm::Function *> Reduction =
5582 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5583 ArrayRef<const Expr *> ReductionOps) {
5592 Args.push_back(&LHSArg);
5593 Args.push_back(&RHSArg);
5596 std::string Name =
getName({
"omp",
"reduction",
"reduction_func"});
5601 Fn->setDoesNotRecurse();
5618 auto IPriv = Privates.begin();
5620 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5621 const auto *RHSVar =
5622 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5623 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5626 const auto *LHSVar =
5627 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5628 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5631 QualType PrivTy = (*IPriv)->getType();
5639 const auto *OVE = cast<OpaqueValueExpr>(VLA->
getSizeExpr());
5646 IPriv = Privates.begin();
5647 auto ILHS = LHSExprs.begin();
5648 auto IRHS = RHSExprs.begin();
5649 for (
const Expr *E : ReductionOps) {
5650 if ((*IPriv)->getType()->isArrayType()) {
5652 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5653 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5655 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5667 Scope.ForceCleanup();
5673 const Expr *ReductionOp,
5674 const Expr *PrivateRef,
5679 const auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
5680 const auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
5682 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
5741 if (SimpleReduction) {
5743 auto IPriv = Privates.begin();
5744 auto ILHS = LHSExprs.begin();
5745 auto IRHS = RHSExprs.begin();
5746 for (
const Expr *E : ReductionOps) {
5748 cast<DeclRefExpr>(*IRHS));
5758 auto Size = RHSExprs.size();
5759 for (
const Expr *E : Privates) {
5764 llvm::APInt ArraySize(32, Size);
5769 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
5770 auto IPriv = Privates.begin();
5772 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5778 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5795 LHSExprs, RHSExprs, ReductionOps);
5798 std::string Name =
getName({
"reduction"});
5811 CGF.
Builder.getInt32(RHSExprs.size()),
5812 ReductionArrayTySize,
5823 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5824 llvm::SwitchInst *SwInst =
5825 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5834 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5843 auto &&CodeGen = [
Privates, LHSExprs, RHSExprs, ReductionOps](
5846 auto IPriv = Privates.begin();
5847 auto ILHS = LHSExprs.begin();
5848 auto IRHS = RHSExprs.begin();
5849 for (
const Expr *E : ReductionOps) {
5850 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5851 cast<DeclRefExpr>(*IRHS));
5858 CommonActionTy Action(
5866 CGF.EmitBranch(DefaultBB);
5873 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(
".omp.reduction.case2");
5874 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5875 CGF.EmitBlock(Case2BB);
5877 auto &&AtomicCodeGen = [Loc,
Privates, LHSExprs, RHSExprs, ReductionOps](
5879 auto ILHS = LHSExprs.begin();
5880 auto IRHS = RHSExprs.begin();
5881 auto IPriv = Privates.begin();
5882 for (
const Expr *E : ReductionOps) {
5883 const Expr *XExpr =
nullptr;
5884 const Expr *EExpr =
nullptr;
5885 const Expr *UpExpr =
nullptr;
5887 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5888 if (BO->getOpcode() == BO_Assign) {
5889 XExpr = BO->getLHS();
5890 UpExpr = BO->getRHS();
5894 const Expr *RHSExpr = UpExpr;
5897 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5901 RHSExpr = ACO->getCond();
5903 if (
const auto *BORHS =
5905 EExpr = BORHS->getRHS();
5906 BO = BORHS->getOpcode();
5910 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5911 auto &&AtomicRedGen = [BO, VD,
5913 const Expr *EExpr,
const Expr *UpExpr) {
5914 LValue X = CGF.EmitLValue(XExpr);
5917 E = CGF.EmitAnyExpr(EExpr);
5918 CGF.EmitOMPAtomicSimpleUpdateExpr(
5920 llvm::AtomicOrdering::Monotonic, Loc,
5921 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5924 VD, [&CGF, VD, XRValue, Loc]() {
5925 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5926 CGF.emitOMPSimpleStore(
5927 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5928 VD->getType().getNonReferenceType(), Loc);
5932 return CGF.EmitAnyExpr(UpExpr);
5935 if ((*IPriv)->getType()->isArrayType()) {
5937 const auto *RHSVar =
5938 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5940 AtomicRedGen, XExpr, EExpr, UpExpr);
5943 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5948 const Expr *,
const Expr *) {
5950 std::string Name = RT.getName({
"atomic_reduction"});
5951 RT.emitCriticalRegion(
5959 if ((*IPriv)->getType()->isArrayType()) {
5960 const auto *LHSVar =
5961 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5962 const auto *RHSVar =
5963 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5967 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5992 CGF.EmitBranch(DefaultBB);
5993 CGF.EmitBlock(DefaultBB,
true);
6001 llvm::raw_svector_ostream Out(Buffer);
6005 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
6008 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.
getMangledName(D)});
6009 Out << Prefix << Name <<
"_" 6010 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
6029 Args.emplace_back(&Param);
6030 const auto &FnInfo =
6037 Fn->setDoesNotRecurse();
6069 llvm::ConstantPointerNull::get(CGM.
VoidPtrTy),
6094 const Expr *ReductionOp,
6096 const Expr *PrivateRef) {
6098 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
6099 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
6105 Args.emplace_back(&ParamInOut);
6106 Args.emplace_back(&ParamIn);
6107 const auto &FnInfo =
6114 Fn->setDoesNotRecurse();
6132 PrivateScope.
addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() {
6140 PrivateScope.
addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() {
6153 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
6154 cast<DeclRefExpr>(RHS));
6176 Args.emplace_back(&Param);
6177 const auto &FnInfo =
6184 Fn->setDoesNotRecurse();
6210 ArrayRef<const Expr *> RHSExprs,
const OMPTaskDataTy &Data) {
6236 llvm::APInt ArraySize(64, Size);
6243 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6246 llvm::ConstantInt::get(
CGM.
SizeTy, Cnt)};
6254 RCG.emitSharedLValue(CGF, Cnt);
6261 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
6268 bool DelayedCreation = !!SizeVal;
6294 if (DelayedCreation) {
6296 llvm::ConstantInt::get(
CGM.
Int32Ty, 1,
true),
6306 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
6367 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
6368 Region->emitUntiedSwitch(CGF);
6377 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
6393 if (CancelRegion == OMPD_parallel)
6394 CancelKind = CancelParallel;
6395 else if (CancelRegion == OMPD_for)
6396 CancelKind = CancelLoop;
6397 else if (CancelRegion == OMPD_sections)
6398 CancelKind = CancelSections;
6400 assert(CancelRegion == OMPD_taskgroup);
6401 CancelKind = CancelTaskgroup;
6413 if (
auto *OMPRegionInfo =
6417 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6430 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6448 if (
auto *OMPRegionInfo =
6450 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](
CodeGenFunction &CGF,
6462 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
6463 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
6464 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6465 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6466 CGF.EmitBlock(ExitBB);
6469 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6470 CGF.EmitBranchThroughCleanup(CancelDest);
6471 CGF.EmitBlock(ContBB,
true);
6485 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6487 assert(!ParentName.empty() &&
"Invalid target region parent name!");
6490 IsOffloadEntry, CodeGen);
6495 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6513 llvm::raw_svector_ostream OS(EntryFnName);
6514 OS <<
"__omp_offloading" << llvm::format(
"_%x", DeviceID)
6515 << llvm::format(
"_%x_", FileID) << ParentName <<
"_l" <<
Line;
6521 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6528 if (!IsOffloadEntry)
6543 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn,
CGM.
Int8PtrTy);
6544 OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
6545 OutlinedFn->setDSOLocal(
false);
6547 std::string Name =
getName({EntryFnName,
"region_id"});
6548 OutlinedFnID =
new llvm::GlobalVariable(
6550 llvm::GlobalValue::WeakAnyLinkage,
6551 llvm::Constant::getNullValue(
CGM.
Int8Ty), Name);
6556 DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
6573 while (
const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) {
6575 for (
const Stmt *S : C->body()) {
6576 if (
const auto *E = dyn_cast<Expr>(S)) {
6581 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
6582 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
6585 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
6586 if (llvm::all_of(DS->decls(), [&Ctx](
const Decl *D) {
6587 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6588 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6589 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6590 isa<UsingDirectiveDecl>(D) ||
6591 isa<OMPDeclareReductionDecl>(D) ||
6592 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6594 const auto *VD = dyn_cast<
VarDecl>(D);
6627 "Clauses associated with the teams directive expected to be emitted " 6628 "only for the host!");
6631 "Expected target-based executable directive.");
6633 switch (DirectiveKind) {
6638 const Stmt *ChildStmt =
6640 if (
const auto *NestedDir =
6641 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6644 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6646 const Expr *NumTeams =
6651 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6654 return Bld.getInt32(0);
6658 return Bld.getInt32(1);
6659 return Bld.getInt32(0);
6663 case OMPD_target_teams:
6664 case OMPD_target_teams_distribute:
6665 case OMPD_target_teams_distribute_simd:
6666 case OMPD_target_teams_distribute_parallel_for:
6667 case OMPD_target_teams_distribute_parallel_for_simd: {
6670 const Expr *NumTeams =
6675 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6678 return Bld.getInt32(0);
6680 case OMPD_target_parallel:
6681 case OMPD_target_parallel_for:
6682 case OMPD_target_parallel_for_simd:
6683 case OMPD_target_simd:
6684 return Bld.getInt32(1);
6687 case OMPD_parallel_for:
6688 case OMPD_parallel_sections:
6690 case OMPD_parallel_for_simd:
6692 case OMPD_cancellation_point:
6694 case OMPD_threadprivate:
6703 case OMPD_taskyield:
6706 case OMPD_taskgroup:
6710 case OMPD_target_data:
6711 case OMPD_target_exit_data:
6712 case OMPD_target_enter_data:
6713 case OMPD_distribute:
6714 case OMPD_distribute_simd:
6715 case OMPD_distribute_parallel_for:
6716 case OMPD_distribute_parallel_for_simd:
6717 case OMPD_teams_distribute:
6718 case OMPD_teams_distribute_simd:
6719 case OMPD_teams_distribute_parallel_for:
6720 case OMPD_teams_distribute_parallel_for_simd:
6721 case OMPD_target_update:
6722 case OMPD_declare_simd:
6723 case OMPD_declare_target:
6724 case OMPD_end_declare_target:
6725 case OMPD_declare_reduction:
6726 case OMPD_declare_mapper:
6728 case OMPD_taskloop_simd:
6733 llvm_unreachable(
"Unexpected directive kind.");
6740 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6747 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6750 for (
const auto *C : Dir->getClausesOfKind<
OMPIfClause>()) {
6752 C->getNameModifier() == OMPD_parallel) {
6762 return CGF.
Builder.getInt32(1);
6765 if (
const auto *PreInit =
6767 for (
const auto *I : PreInit->decls()) {
6768 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6784 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6786 const auto *NumThreadsClause =
6789 CGF, NumThreadsClause->getNumThreads()->getSourceRange());
6790 if (
const auto *PreInit =
6791 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6792 for (
const auto *I : PreInit->decls()) {
6793 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6802 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads());
6805 if (DefaultThreadLimitVal)
6806 NumThreads = CGF.
Builder.CreateSelect(
6807 CGF.
Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads),
6808 DefaultThreadLimitVal, NumThreads);
6810 NumThreads = DefaultThreadLimitVal ? DefaultThreadLimitVal
6815 NumThreads = CGF.
Builder.CreateSelect(CondVal, NumThreads,
6821 return CGF.
Builder.getInt32(1);
6822 return DefaultThreadLimitVal;
6824 return DefaultThreadLimitVal ? DefaultThreadLimitVal
6840 "Clauses associated with the teams directive expected to be emitted " 6841 "only for the host!");
6844 "Expected target-based executable directive.");
6848 switch (DirectiveKind) {
6855 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6857 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6859 const auto *ThreadLimitClause =
6862 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6863 if (
const auto *PreInit =
6864 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6865 for (
const auto *I : PreInit->decls()) {
6866 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6876 ThreadLimitClause->getThreadLimit(),
true);
6878 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6882 CS = Dir->getInnermostCapturedStmt();
6885 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6889 CS = Dir->getInnermostCapturedStmt();
6894 return Bld.getInt32(1);
6896 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
6898 case OMPD_target_teams: {
6903 ThreadLimitClause->getThreadLimit(),
true);
6905 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6912 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6913 if (Dir->getDirectiveKind() == OMPD_distribute) {
6914 CS = Dir->getInnermostCapturedStmt();
6919 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
6921 case OMPD_target_teams_distribute:
6926 ThreadLimitClause->getThreadLimit(),
true);
6928 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6931 case OMPD_target_parallel:
6932 case OMPD_target_parallel_for:
6933 case OMPD_target_parallel_for_simd:
6934 case OMPD_target_teams_distribute_parallel_for:
6935 case OMPD_target_teams_distribute_parallel_for_simd: {
6943 C->getNameModifier() == OMPD_parallel) {
6953 return Bld.getInt32(1);
6964 ThreadLimitClause->getThreadLimit(),
true);
6966 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6972 NumThreadsClause->getNumThreads(),
true);
6974 Bld.CreateIntCast(NumThreads, CGF.
Int32Ty,
false);
6975 ThreadLimitVal = ThreadLimitVal
6976 ? Bld.CreateSelect(Bld.CreateICmpULT(NumThreadsVal,
6978 NumThreadsVal, ThreadLimitVal)
6981 if (!ThreadLimitVal)
6982 ThreadLimitVal = Bld.getInt32(0);
6984 return Bld.CreateSelect(CondVal, ThreadLimitVal, Bld.getInt32(1));
6985 return ThreadLimitVal;
6987 case OMPD_target_teams_distribute_simd:
6988 case OMPD_target_simd:
6989 return Bld.getInt32(1);
6992 case OMPD_parallel_for:
6993 case OMPD_parallel_sections:
6995 case OMPD_parallel_for_simd:
6997 case OMPD_cancellation_point:
6999 case OMPD_threadprivate:
7008 case OMPD_taskyield:
7011 case OMPD_taskgroup:
7015 case OMPD_target_data:
7016 case OMPD_target_exit_data:
7017 case OMPD_target_enter_data:
7018 case OMPD_distribute:
7019 case OMPD_distribute_simd:
7020 case OMPD_distribute_parallel_for:
7021 case OMPD_distribute_parallel_for_simd:
7022 case OMPD_teams_distribute:
7023 case OMPD_teams_distribute_simd:
7024 case OMPD_teams_distribute_parallel_for:
7025 case OMPD_teams_distribute_parallel_for_simd:
7026 case OMPD_target_update:
7027 case OMPD_declare_simd:
7028 case OMPD_declare_target:
7029 case OMPD_end_declare_target:
7030 case OMPD_declare_reduction:
7031 case OMPD_declare_mapper:
7033 case OMPD_taskloop_simd:
7038 llvm_unreachable(
"Unsupported directive kind.");
7048 class MappableExprsHandler {
7052 enum OpenMPOffloadMappingFlags : uint64_t {
7058 OMP_MAP_FROM = 0x02,
7061 OMP_MAP_ALWAYS = 0x04,
7064 OMP_MAP_DELETE = 0x08,
7067 OMP_MAP_PTR_AND_OBJ = 0x10,
7070 OMP_MAP_TARGET_PARAM = 0x20,
7074 OMP_MAP_RETURN_PARAM = 0x40,
7077 OMP_MAP_PRIVATE = 0x80,
7079 OMP_MAP_LITERAL = 0x100,
7081 OMP_MAP_IMPLICIT = 0x200,
7084 OMP_MAP_MEMBER_OF = 0xffff000000000000,
7085 LLVM_MARK_AS_BITMASK_ENUM( OMP_MAP_MEMBER_OF),
7090 class BasePointerInfo {
7099 : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
7101 const ValueDecl *getDevicePtrDecl()
const {
return DevPtrDecl; }
7102 void setDevicePtrDecl(
const ValueDecl *D) { DevPtrDecl = D; }
7113 struct StructRangeInfoTy {
7114 std::pair<
unsigned ,
Address > LowestElem = {
7116 std::pair<
unsigned ,
Address > HighestElem = {
7126 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7127 bool ReturnDevicePointer =
false;
7128 bool IsImplicit =
false;
7130 MapInfo() =
default;
7134 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7135 bool ReturnDevicePointer,
bool IsImplicit)
7136 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7137 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {}
7143 struct DeferredDevicePtrEntryTy {
7144 const Expr *IE =
nullptr;
7147 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD)
7160 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
7179 if (
const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
7181 OAE->getBase()->IgnoreParenImpCasts())
7182 .getCanonicalType();
7186 if (!OAE->getLength() && OAE->getColonLoc().isValid())
7191 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
7193 const auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
7194 assert(ATy &&
"Expecting array type if not a pointer type.");
7195 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
7200 if (!OAE->getLength())
7206 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
7215 OpenMPOffloadMappingFlags getMapTypeBits(
7217 bool IsImplicit,
bool AddPtrFlag,
bool AddIsTargetParamFlag)
const {
7218 OpenMPOffloadMappingFlags Bits =
7219 IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE;
7221 case OMPC_MAP_alloc:
7222 case OMPC_MAP_release:
7232 Bits |= OMP_MAP_FROM;
7234 case OMPC_MAP_tofrom:
7235 Bits |= OMP_MAP_TO | OMP_MAP_FROM;
7237 case OMPC_MAP_delete:
7238 Bits |= OMP_MAP_DELETE;
7241 llvm_unreachable(
"Unexpected map type!");
7244 Bits |= OMP_MAP_PTR_AND_OBJ;
7245 if (AddIsTargetParamFlag)
7246 Bits |= OMP_MAP_TARGET_PARAM;
7247 if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_always)
7248 != MapModifiers.end())
7249 Bits |= OMP_MAP_ALWAYS;
7255 bool isFinalArraySectionExpression(
const Expr *E)
const {
7263 if (OASE->getColonLoc().isInvalid())
7266 const Expr *Length = OASE->getLength();
7273 OASE->getBase()->IgnoreParenImpCasts())
7274 .getCanonicalType();
7275 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
7276 return ATy->getSize().getSExtValue() != 1;
7288 llvm::APSInt ConstLength = Result.
Val.
getInt();
7289 return ConstLength.getSExtValue() != 1;
7296 void generateInfoForComponentList(
7298 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7300 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
7301 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
7302 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
7304 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7463 bool IsCaptureFirstInfo = IsFirstComponentList;
7467 bool RequiresReference =
false;
7470 auto CI = Components.rbegin();
7471 auto CE = Components.rend();
7476 bool IsExpressionFirstInfo =
true;
7478 const Expr *AssocExpr = I->getAssociatedExpression();
7482 if (isa<MemberExpr>(AssocExpr)) {
7486 }
else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7488 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7494 if (
const auto *VD =
7495 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7497 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7498 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7499 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
7501 RequiresReference =
true;
7511 I->getAssociatedDeclaration()->getType().getNonReferenceType();
7535 bool ShouldBeMemberOf =
false;
7546 for (; I != CE; ++I) {
7548 if (!EncounteredME) {
7549 EncounteredME = dyn_cast<
MemberExpr>(I->getAssociatedExpression());
7553 ShouldBeMemberOf =
true;
7556 auto Next = std::next(I);
7564 bool IsFinalArraySection =
7565 isFinalArraySectionExpression(I->getAssociatedExpression());
7578 if (Next == CE || IsPointer || IsFinalArraySection) {
7581 assert((Next == CE ||
7582 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7583 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7584 isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) &&
7585 "Unexpected expression");
7593 bool IsMemberPointer =
7594 IsPointer && EncounteredME &&
7595 (dyn_cast<
MemberExpr>(I->getAssociatedExpression()) ==
7597 if (!OverlappedElements.empty()) {
7599 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
7600 assert(Next == CE &&
7601 "Expected last element for the overlapped elements.");
7602 assert(!IsPointer &&
7603 "Unexpected base element with the pointer type.");
7606 PartialStruct.LowestElem = {0, LB};
7608 I->getAssociatedExpression()->getType());
7613 PartialStruct.HighestElem = {
7614 std::numeric_limits<decltype(
7615 PartialStruct.HighestElem.first)>
::max(),
7617 PartialStruct.Base = BP;
7619 OpenMPOffloadMappingFlags Flags =
7621 getMapTypeBits(MapType, MapModifiers, IsImplicit,
7628 Component : OverlappedElements) {
7632 if (MC.getAssociatedDeclaration()) {
7636 Size = CGF.
Builder.CreatePtrDiff(
7643 Pointers.push_back(LB.getPointer());
7646 Types.push_back(Flags);
7650 Pointers.push_back(LB.getPointer());
7651 Size = CGF.
Builder.CreatePtrDiff(
7657 Types.push_back(Flags);
7660 llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());
7661 if (!IsMemberPointer) {
7663 Pointers.push_back(LB.getPointer());
7671 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7672 MapType, MapModifiers, IsImplicit,
7673 !IsExpressionFirstInfo || RequiresReference,
7674 IsCaptureFirstInfo && !RequiresReference);
7676 if (!IsExpressionFirstInfo) {
7680 Flags &= ~(OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_ALWAYS |
7683 if (ShouldBeMemberOf) {
7686 Flags |= OMP_MAP_MEMBER_OF;
7689 ShouldBeMemberOf =
false;
7693 Types.push_back(Flags);
7699 if (EncounteredME) {
7700 const auto *FD = dyn_cast<
FieldDecl>(EncounteredME->getMemberDecl());
7704 if (!PartialStruct.Base.isValid()) {
7705 PartialStruct.LowestElem = {FieldIndex, LB};
7706 PartialStruct.HighestElem = {FieldIndex, LB};
7707 PartialStruct.Base = BP;
7708 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
7709 PartialStruct.LowestElem = {FieldIndex, LB};
7710 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
7711 PartialStruct.HighestElem = {FieldIndex, LB};
7716 if (IsFinalArraySection)
7723 IsExpressionFirstInfo =
false;
7724 IsCaptureFirstInfo =
false;
7732 MappableExprsHandler::OpenMPOffloadMappingFlags
7742 return MappableExprsHandler::OMP_MAP_ALWAYS |
7743 MappableExprsHandler::OMP_MAP_TO;
7745 return MappableExprsHandler::OMP_MAP_TO |
7746 MappableExprsHandler::OMP_MAP_PTR_AND_OBJ;
7747 return MappableExprsHandler::OMP_MAP_PRIVATE |
7748 MappableExprsHandler::OMP_MAP_TO;
7750 return MappableExprsHandler::OMP_MAP_TO |
7751 MappableExprsHandler::OMP_MAP_FROM;
7754 static OpenMPOffloadMappingFlags getMemberOfFlag(
unsigned Position) {
7756 return static_cast<OpenMPOffloadMappingFlags
>(((uint64_t)Position + 1)
7760 static void setCorrectMemberOfFlag(OpenMPOffloadMappingFlags &Flags,
7761 OpenMPOffloadMappingFlags MemberOfFlag) {
7765 if ((Flags & OMP_MAP_PTR_AND_OBJ) &&
7766 ((Flags & OMP_MAP_MEMBER_OF) != OMP_MAP_MEMBER_OF))
7771 Flags &= ~OMP_MAP_MEMBER_OF;
7772 Flags |= MemberOfFlag;
7777 bool AsBase)
const {
7780 llvm::StructType *St =
7783 unsigned NumElements = St->getNumElements();
7785 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7786 RecordLayout(NumElements);
7789 for (
const auto &I : RD->
bases()) {
7792 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7801 RecordLayout[FieldIndex] =
Base;
7804 for (
const auto &I : RD->
vbases()) {
7805 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7807 if (
Base->isEmpty())
7810 if (RecordLayout[FieldIndex])
7812 RecordLayout[FieldIndex] =
Base;
7815 assert(!RD->
isUnion() &&
"Unexpected union.");
7816 for (
const auto *Field : RD->
fields()) {
7819 if (!Field->isBitField() && !Field->isZeroSize(CGF.
getContext())) {
7821 RecordLayout[FieldIndex] = Field;
7824 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7825 &Data : RecordLayout) {
7829 getPlainLayout(
Base, Layout,
true);
7831 Layout.push_back(Data.get<
const FieldDecl *>());
7837 : CurDir(Dir), CGF(CGF) {
7840 for (
const auto *D : C->varlists())
7841 FirstPrivateDecls.try_emplace(
7842 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()), C->isImplicit());
7845 for (
auto L : C->component_lists())
7846 DevPointersMap[L.first].push_back(L.second);
7852 void emitCombinedEntry(MapBaseValuesArrayTy &BasePointers,
7853 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7854 MapFlagsArrayTy &Types, MapFlagsArrayTy &CurTypes,
7855 const StructRangeInfoTy &PartialStruct)
const {
7857 BasePointers.push_back(PartialStruct.Base.getPointer());
7859 llvm::Value *LB = PartialStruct.LowestElem.second.getPointer();
7860 Pointers.push_back(LB);
7862 llvm::Value *HB = PartialStruct.HighestElem.second.getPointer();
7869 Sizes.push_back(Size);
7871 Types.push_back(OMP_MAP_TARGET_PARAM);
7873 (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM;
7878 OpenMPOffloadMappingFlags MemberOfFlag =
7879 getMemberOfFlag(BasePointers.size() - 1);
7880 for (
auto &M : CurTypes)
7881 setCorrectMemberOfFlag(M, MemberOfFlag);
7888 void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
7889 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7890 MapFlagsArrayTy &Types)
const {
7894 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
7898 auto &&InfoGen = [&Info](
7902 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7903 bool ReturnDevicePointer,
bool IsImplicit) {
7904 const ValueDecl *VD =
7906 Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
7912 for (
const auto &L : C->component_lists()) {
7913 InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifiers(),
7914 false, C->isImplicit());
7917 for (
const auto &L : C->component_lists()) {
7918 InfoGen(L.first, L.second, OMPC_MAP_to,
llvm::None,
7919 false, C->isImplicit());
7922 for (
const auto &L : C->component_lists()) {
7923 InfoGen(L.first, L.second, OMPC_MAP_from,
llvm::None,
7924 false, C->isImplicit());
7933 llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
7937 for (
const auto *C :
7939 for (
const auto &L : C->component_lists()) {
7940 assert(!L.second.empty() &&
"Not expecting empty list of components!");
7941 const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
7942 VD = cast<ValueDecl>(VD->getCanonicalDecl());
7943 const Expr *IE = L.second.back().getAssociatedExpression();
7947 auto It = Info.find(isa<MemberExpr>(IE) ?
nullptr : VD);
7951 if (It != Info.end()) {
7952 auto CI = std::find_if(
7953 It->second.begin(), It->second.end(), [VD](
const MapInfo &MI) {
7954 return MI.Components.back().getAssociatedDeclaration() == VD;
7958 if (CI != It->second.end()) {
7959 CI->ReturnDevicePointer =
true;
7968 if (isa<MemberExpr>(IE)) {
7977 false, C->isImplicit());
7978 DeferredInfo[
nullptr].emplace_back(IE, VD);
7982 BasePointers.emplace_back(Ptr, VD);
7983 Pointers.push_back(Ptr);
7984 Sizes.push_back(llvm::Constant::getNullValue(this->CGF.
Int64Ty));
7985 Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM);
7990 for (
const auto &M : Info) {
7993 bool IsFirstComponentList =
true;
7996 MapBaseValuesArrayTy CurBasePointers;
7997 MapValuesArrayTy CurPointers;
7998 MapValuesArrayTy CurSizes;
7999 MapFlagsArrayTy CurTypes;
8000 StructRangeInfoTy PartialStruct;
8002 for (
const MapInfo &L : M.second) {
8003 assert(!L.Components.empty() &&
8004 "Not expecting declaration with no component lists.");
8007 unsigned CurrentBasePointersIdx = CurBasePointers.size();
8009 this->generateInfoForComponentList(
8010 L.MapType, L.MapModifiers, L.Components, CurBasePointers,
8011 CurPointers, CurSizes, CurTypes, PartialStruct,
8012 IsFirstComponentList, L.IsImplicit);
8016 if (L.ReturnDevicePointer) {
8017 assert(CurBasePointers.size() > CurrentBasePointersIdx &&
8018 "Unexpected number of mapped base pointers.");
8020 const ValueDecl *RelevantVD =
8021 L.Components.back().getAssociatedDeclaration();
8022 assert(RelevantVD &&
8023 "No relevant declaration related with device pointer??");
8025 CurBasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
8026 CurTypes[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM;
8028 IsFirstComponentList =
false;
8033 auto CI = DeferredInfo.find(M.first);
8034 if (CI != DeferredInfo.end()) {
8035 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8038 this->CGF.
EmitLValue(L.IE), L.IE->getExprLoc());
8039 CurBasePointers.emplace_back(BasePtr, L.VD);
8040 CurPointers.push_back(Ptr);
8041 CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8045 CurTypes.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_RETURN_PARAM |
8052 if (PartialStruct.Base.isValid())
8053 emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
8057 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8058 Pointers.append(CurPointers.begin(), CurPointers.end());
8059 Sizes.append(CurSizes.begin(), CurSizes.end());
8060 Types.append(CurTypes.begin(), CurTypes.end());
8065 void generateInfoForLambdaCaptures(
8066 const ValueDecl *VD,
llvm::Value *Arg, MapBaseValuesArrayTy &BasePointers,
8067 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
8068 MapFlagsArrayTy &Types,
8069 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
8070 const auto *RD = VD->
getType()
8079 llvm::DenseMap<const VarDecl *, FieldDecl *> Captures;
8086 LambdaPointers.try_emplace(ThisLVal.
getPointer(), VDLVal.getPointer());
8087 BasePointers.push_back(ThisLVal.
getPointer());
8088 Pointers.push_back(ThisLValVal.
getPointer());
8092 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8093 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8096 if (!LC.capturesVariable())
8098 const VarDecl *VD = LC.getCapturedVar();
8101 auto It = Captures.find(VD);
8102 assert(It != Captures.end() &&
"Found lambda capture without field.");
8106 LambdaPointers.try_emplace(VarLVal.getPointer(), VDLVal.getPointer());
8107 BasePointers.push_back(VarLVal.getPointer());
8109 Sizes.push_back(CGF.
Builder.CreateIntCast(
8115 LambdaPointers.try_emplace(VarLVal.getPointer(), VDLVal.getPointer());
8116 BasePointers.push_back(VarLVal.getPointer());
8118 Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
8120 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8121 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8126 void adjustMemberOfForLambdaCaptures(
8127 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8128 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8129 MapFlagsArrayTy &Types)
const {
8130 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
8132 if (Types[I] != (OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8133 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT))
8135 llvm::Value *BasePtr = LambdaPointers.lookup(*BasePointers[I]);
8136 assert(BasePtr &&
"Unable to find base lambda address.");
8138 for (
unsigned J = I; J > 0; --J) {
8139 unsigned Idx = J - 1;
8140 if (Pointers[Idx] != BasePtr)
8145 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
8149 OpenMPOffloadMappingFlags MemberOfFlag = getMemberOfFlag(TgtIdx);
8150 setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8158 MapBaseValuesArrayTy &BasePointers,
8159 MapValuesArrayTy &Pointers,
8160 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
8161 StructRangeInfoTy &PartialStruct)
const {
8163 "Not expecting to generate map info for a variable array type!");
8173 if (DevPointersMap.count(VD)) {
8174 BasePointers.emplace_back(Arg, VD);
8175 Pointers.push_back(Arg);
8179 Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM);
8189 for (
const auto &L : C->decl_component_lists(VD)) {
8190 assert(L.first == VD &&
8191 "We got information for the wrong declaration??");
8192 assert(!L.second.empty() &&
8193 "Not expecting declaration with no component lists.");
8194 DeclComponentLists.emplace_back(L.second, C->getMapType(),
8195 C->getMapTypeModifiers(),
8201 llvm::SmallDenseMap<
8208 for (
const MapData &L : DeclComponentLists) {
8209 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8210 OpenMPMapClauseKind MapType;
8211 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8213 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8215 for (
const MapData &L1 : makeArrayRef(DeclComponentLists).slice(Count)) {
8216 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
8217 std::tie(Components1, MapType, MapModifiers, IsImplicit) = L1;
8218 auto CI = Components.rbegin();
8219 auto CE = Components.rend();
8220 auto SI = Components1.rbegin();
8221 auto SE = Components1.rend();
8222 for (; CI != CE && SI != SE; ++CI, ++SI) {
8223 if (CI->getAssociatedExpression()->getStmtClass() !=
8224 SI->getAssociatedExpression()->getStmtClass())
8227 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8232 if (CI == CE || SI == SE) {
8233 assert((CI != CE || SI != SE) &&
8234 "Unexpected full match of the mapping components.");
8235 const MapData &BaseData = CI == CE ? L : L1;
8236 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
8237 SI == SE ? Components : Components1;
8238 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8239 OverlappedElements.getSecond().push_back(SubData);
8245 if (!OverlappedData.empty()) {
8246 if (
const auto *CRD =
8247 VD->getType().getCanonicalType()->getAsCXXRecordDecl())
8248 getPlainLayout(CRD, Layout,
false);
8250 const auto *RD = VD->getType().getCanonicalType()->getAsRecordDecl();
8254 for (
auto &Pair : OverlappedData) {
8258 OMPClauseMappableExprCommon::MappableExprComponentListRef
First,
8259 OMPClauseMappableExprCommon::MappableExprComponentListRef
8261 auto CI = First.rbegin();
8262 auto CE = First.rend();
8263 auto SI = Second.rbegin();
8264 auto SE = Second.rend();
8265 for (; CI != CE && SI != SE; ++CI, ++SI) {
8266 if (CI->getAssociatedExpression()->getStmtClass() !=
8267 SI->getAssociatedExpression()->getStmtClass())
8270 if (CI->getAssociatedDeclaration() !=
8271 SI->getAssociatedDeclaration())
8276 if (CI == CE && SI == SE)
8280 if (CI == CE || SI == SE)
8283 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8284 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8285 if (FD1->getParent() == FD2->getParent())
8286 return FD1->getFieldIndex() < FD2->getFieldIndex();
8288 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
8289 return FD == FD1 || FD == FD2;
8297 for (
const auto &Pair : OverlappedData) {
8298 const MapData &L = *Pair.getFirst();
8299 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8300 OpenMPMapClauseKind MapType;
8301 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8303 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8304 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
8305 OverlappedComponents = Pair.getSecond();
8306 bool IsFirstComponentList =
true;
8307 generateInfoForComponentList(MapType, MapModifiers, Components,
8308 BasePointers, Pointers, Sizes, Types,
8309 PartialStruct, IsFirstComponentList,
8310 IsImplicit, OverlappedComponents);
8313 bool IsFirstComponentList = OverlappedData.empty();
8314 for (
const MapData &L : DeclComponentLists) {
8315 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8316 OpenMPMapClauseKind MapType;
8317 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8319 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8320 auto It = OverlappedData.find(&L);
8321 if (It == OverlappedData.end())
8322 generateInfoForComponentList(MapType, MapModifiers, Components,
8323 BasePointers, Pointers, Sizes, Types,
8324 PartialStruct, IsFirstComponentList,
8326 IsFirstComponentList =
false;
8332 void generateInfoForDeclareTargetLink(MapBaseValuesArrayTy &BasePointers,
8333 MapValuesArrayTy &Pointers,
8334 MapValuesArrayTy &Sizes,
8335 MapFlagsArrayTy &Types)
const {
8339 for (
const auto &L : C->component_lists()) {
8342 const auto *VD = dyn_cast<
VarDecl>(L.first);
8346 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
8348 !Res || *Res != OMPDeclareTargetDeclAttr::MT_Link)
8350 StructRangeInfoTy PartialStruct;
8351 generateInfoForComponentList(
8352 C->getMapType(), C->getMapTypeModifiers(), L.second, BasePointers,
8353 Pointers, Sizes, Types, PartialStruct,
8354 true, C->isImplicit());
8355 assert(!PartialStruct.Base.isValid() &&
8356 "No partial structs for declare target link expected.");
8365 MapBaseValuesArrayTy &CurBasePointers,
8366 MapValuesArrayTy &CurPointers,
8367 MapValuesArrayTy &CurSizes,
8368 MapFlagsArrayTy &CurMapTypes)
const {
8369 bool IsImplicit =
true;
8372 CurBasePointers.push_back(CV);
8373 CurPointers.push_back(CV);
8379 CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
8381 CurBasePointers.push_back(CV);
8382 CurPointers.push_back(CV);
8386 CurMapTypes.push_back(OMP_MAP_LITERAL);
8387 CurSizes.push_back(CGF.
Builder.CreateIntCast(
8392 CurMapTypes.push_back(OMP_MAP_NONE);
8393 CurSizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
8396 auto I = FirstPrivateDecls.find(VD);
8397 if (I != FirstPrivateDecls.end())
8398 IsImplicit = I->getSecond();
8403 CurSizes.push_back(CGF.
Builder.CreateIntCast(
8408 CurMapTypes.push_back(getMapModifiersForPrivateClauses(CI));
8410 auto I = FirstPrivateDecls.find(VD);
8411 if (I != FirstPrivateDecls.end() &&
8413 llvm::Constant *Addr =
8419 CurSizes.back(),
false);
8421 CurBasePointers.push_back(Addr);
8422 CurPointers.push_back(Addr);
8424 CurBasePointers.push_back(CV);
8425 if (I != FirstPrivateDecls.end() && ElementType->isAnyPointerType()) {
8431 CurPointers.push_back(CV);
8434 if (I != FirstPrivateDecls.end())
8435 IsImplicit = I->getSecond();
8438 CurMapTypes.back() |= OMP_MAP_TARGET_PARAM;
8442 CurMapTypes.back() |= OMP_MAP_IMPLICIT;
8456 CGOpenMPRuntime::TargetDataInfo &Info) {
8461 Info.clearArrayInfo();
8462 Info.NumberOfPtrs = BasePointers.size();
8464 if (Info.NumberOfPtrs) {
8467 bool hasRuntimeEvaluationCaptureSize =
false;
8469 if (!isa<llvm::Constant>(S)) {
8470 hasRuntimeEvaluationCaptureSize =
true;
8474 llvm::APInt PointerNumAP(32, Info.NumberOfPtrs,
true);
8479 Info.BasePointersArray =
8481 Info.PointersArray =
8489 if (hasRuntimeEvaluationCaptureSize) {
8500 ConstSizes.push_back(cast<llvm::Constant>(S));
8502 auto *SizesArrayInit = llvm::ConstantArray::get(
8503 llvm::ArrayType::get(CGM.
Int64Ty, ConstSizes.size()), ConstSizes);
8505 auto *SizesArrayGbl =
new llvm::GlobalVariable(
8506 CGM.
getModule(), SizesArrayInit->getType(),
8507 true, llvm::GlobalValue::PrivateLinkage,
8508 SizesArrayInit, Name);
8509 SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8510 Info.SizesArray = SizesArrayGbl;
8516 llvm::copy(MapTypes, Mapping.begin());
8517 llvm::Constant *MapTypesArrayInit =
8518 llvm::ConstantDataArray::get(CGF.
Builder.getContext(), Mapping);
8519 std::string MaptypesName =
8521 auto *MapTypesArrayGbl =
new llvm::GlobalVariable(
8522 CGM.
getModule(), MapTypesArrayInit->getType(),
8523 true, llvm::GlobalValue::PrivateLinkage,
8524 MapTypesArrayInit, MaptypesName);
8525 MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8526 Info.MapTypesArray = MapTypesArrayGbl;
8528 for (
unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
8531 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8532 Info.BasePointersArray, 0, I);
8534 BP, BPVal->getType()->getPointerTo(0));
8538 if (Info.requiresDevicePointerInfo())
8539 if (
const ValueDecl *DevVD = BasePointers[I].getDevicePtrDecl())
8540 Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
8544 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8545 Info.PointersArray, 0, I);
8547 P, PVal->getType()->getPointerTo(0));
8551 if (hasRuntimeEvaluationCaptureSize) {
8553 llvm::ArrayType::get(CGM.
Int64Ty, Info.NumberOfPtrs),
8570 llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
8572 if (Info.NumberOfPtrs) {
8574 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8575 Info.BasePointersArray,
8578 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8583 llvm::ArrayType::get(CGM.
Int64Ty, Info.NumberOfPtrs), Info.SizesArray,
8586 llvm::ArrayType::get(CGM.
Int64Ty, Info.NumberOfPtrs),
8591 BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
8592 PointersArrayArg = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
8593 SizesArrayArg = llvm::ConstantPointerNull::get(CGM.
Int64Ty->getPointerTo());
8595 llvm::ConstantPointerNull::get(CGM.
Int64Ty->getPointerTo());
8605 const Stmt *ChildStmt =
8608 if (
const auto *NestedDir =
8609 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8615 if (DKind == OMPD_teams) {
8616 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8621 if (
const auto *NND =
8622 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8623 DKind = NND->getDirectiveKind();
8629 case OMPD_target_teams:
8633 case OMPD_target_parallel:
8634 case OMPD_target_simd:
8635 case OMPD_target_parallel_for:
8636 case OMPD_target_parallel_for_simd:
8638 case OMPD_target_teams_distribute:
8639 case OMPD_target_teams_distribute_simd:
8640 case OMPD_target_teams_distribute_parallel_for:
8641 case OMPD_target_teams_distribute_parallel_for_simd:
8644 case OMPD_parallel_for:
8645 case OMPD_parallel_sections:
8647 case OMPD_parallel_for_simd:
8649 case OMPD_cancellation_point:
8651 case OMPD_threadprivate:
8660 case OMPD_taskyield:
8663 case OMPD_taskgroup:
8667 case OMPD_target_data:
8668 case OMPD_target_exit_data:
8669 case OMPD_target_enter_data:
8670 case OMPD_distribute:
8671 case OMPD_distribute_simd:
8672 case OMPD_distribute_parallel_for:
8673 case OMPD_distribute_parallel_for_simd:
8674 case OMPD_teams_distribute:
8675 case OMPD_teams_distribute_simd:
8676 case OMPD_teams_distribute_parallel_for:
8677 case OMPD_teams_distribute_parallel_for_simd:
8678 case OMPD_target_update:
8679 case OMPD_declare_simd:
8680 case OMPD_declare_target:
8681 case OMPD_end_declare_target:
8682 case OMPD_declare_reduction:
8683 case OMPD_declare_mapper:
8685 case OMPD_taskloop_simd:
8688 llvm_unreachable(
"Unexpected directive.");
8706 const auto *LD = cast<OMPLoopDirective>(TD);
8707 auto &&CodeGen = [LD, &Device, &SizeEmitter,
this](
CodeGenFunction &CGF,
8709 llvm::Value *NumIterations = SizeEmitter(CGF, *LD);
8714 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
8717 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
8720 CGF.EmitRuntimeCall(
8728 llvm::Function *OutlinedFn,
8730 const Expr *IfCond,
const Expr *Device) {
8734 assert(OutlinedFn &&
"Invalid outlined function!");
8741 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
8748 auto &&ThenGen = [
this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
8749 &MapTypesArray, &CS, RequiresOuterTask,
8761 assert(OutlinedFnID &&
"Invalid outlined function ID!");
8766 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
8769 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
8774 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
8816 assert(NumThreads &&
"Thread limit expression should be available along " 8817 "with number of teams.");
8821 InputInfo.BasePointersArray.getPointer(),
8822 InputInfo.PointersArray.getPointer(),
8823 InputInfo.SizesArray.getPointer(),
8827 Return = CGF.EmitRuntimeCall(
8835 InputInfo.BasePointersArray.getPointer(),
8836 InputInfo.PointersArray.getPointer(),
8837 InputInfo.SizesArray.getPointer(),
8839 Return = CGF.EmitRuntimeCall(
8846 llvm::BasicBlock *OffloadFailedBlock =
8847 CGF.createBasicBlock(
"omp_offload.failed");
8848 llvm::BasicBlock *OffloadContBlock =
8849 CGF.createBasicBlock(
"omp_offload.cont");
8850 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return);
8851 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
8853 CGF.EmitBlock(OffloadFailedBlock);
8854 if (RequiresOuterTask) {
8855 CapturedVars.clear();
8856 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
8859 CGF.EmitBranch(OffloadContBlock);
8861 CGF.EmitBlock(OffloadContBlock,
true);
8865 auto &&ElseGen = [
this, &D, OutlinedFn, &CS, &CapturedVars,
8868 if (RequiresOuterTask) {
8869 CapturedVars.clear();
8870 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
8875 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
8876 &CapturedVars, RequiresOuterTask,
8885 MappableExprsHandler MEHandler(D, CGF);
8886 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
8888 auto RI = CS.getCapturedRecordDecl()->field_begin();
8889 auto CV = CapturedVars.begin();
8891 CE = CS.capture_end();
8892 CI != CE; ++CI, ++RI, ++CV) {
8897 MappableExprsHandler::StructRangeInfoTy PartialStruct;
8901 if (CI->capturesVariableArrayType()) {
8902 CurBasePointers.push_back(*CV);
8903 CurPointers.push_back(*CV);
8904 CurSizes.push_back(CGF.Builder.CreateIntCast(
8905 CGF.getTypeSize(RI->getType()), CGF.Int64Ty,
true));
8907 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL |
8908 MappableExprsHandler::OMP_MAP_TARGET_PARAM |
8909 MappableExprsHandler::OMP_MAP_IMPLICIT);
8913 MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
8914 CurSizes, CurMapTypes, PartialStruct);
8915 if (CurBasePointers.empty())
8916 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
8917 CurPointers, CurSizes, CurMapTypes);
8920 if (CI->capturesVariable())
8921 MEHandler.generateInfoForLambdaCaptures(
8922 CI->getCapturedVar(), *CV, CurBasePointers, CurPointers, CurSizes,
8923 CurMapTypes, LambdaPointers);
8926 assert(!CurBasePointers.empty() &&
8927 "Non-existing map pointer for capture!");
8928 assert(CurBasePointers.size() == CurPointers.size() &&
8929 CurBasePointers.size() == CurSizes.size() &&
8930 CurBasePointers.size() == CurMapTypes.size() &&
8931 "Inconsistent map information sizes!");
8935 if (PartialStruct.Base.isValid())
8936 MEHandler.emitCombinedEntry(BasePointers, Pointers, Sizes, MapTypes,
8937 CurMapTypes, PartialStruct);
8940 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8941 Pointers.append(CurPointers.begin(), CurPointers.end());
8942 Sizes.append(CurSizes.begin(), CurSizes.end());
8943 MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
8946 MEHandler.adjustMemberOfForLambdaCaptures(LambdaPointers, BasePointers,
8947 Pointers, MapTypes);
8950 MEHandler.generateInfoForDeclareTargetLink(BasePointers, Pointers, Sizes,
8960 InputInfo.BasePointersArray =
8962 InputInfo.PointersArray =
8966 if (RequiresOuterTask)
8967 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
8972 auto &&TargetElseGen = [
this, &ElseGen, &D, RequiresOuterTask](
8974 if (RequiresOuterTask) {
8976 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
9000 StringRef ParentName) {
9005 bool RequiresDeviceCodegen =
9006 isa<OMPExecutableDirective>(S) &&
9008 cast<OMPExecutableDirective>(S)->getDirectiveKind());
9010 if (RequiresDeviceCodegen) {
9011 const auto &E = *cast<OMPExecutableDirective>(S);
9024 switch (E.getDirectiveKind()) {
9027 cast<OMPTargetDirective>(E));
9029 case OMPD_target_parallel:
9031 CGM, ParentName, cast<OMPTargetParallelDirective>(E));
9033 case OMPD_target_teams:
9035 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
9037 case OMPD_target_teams_distribute:
9039 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
9041 case OMPD_target_teams_distribute_simd:
9043 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
9045 case OMPD_target_parallel_for:
9047 CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
9049 case OMPD_target_parallel_for_simd:
9051 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
9053 case OMPD_target_simd:
9055 CGM, ParentName, cast<OMPTargetSimdDirective>(E));
9057 case OMPD_target_teams_distribute_parallel_for:
9060 cast<OMPTargetTeamsDistributeParallelForDirective>(E));
9062 case OMPD_target_teams_distribute_parallel_for_simd:
9066 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
9070 case OMPD_parallel_for:
9071 case OMPD_parallel_sections:
9073 case OMPD_parallel_for_simd:
9075 case OMPD_cancellation_point:
9077 case OMPD_threadprivate:
9086 case OMPD_taskyield:
9089 case OMPD_taskgroup:
9093 case OMPD_target_data:
9094 case OMPD_target_exit_data:
9095 case OMPD_target_enter_data:
9096 case OMPD_distribute:
9097 case OMPD_distribute_simd:
9098 case OMPD_distribute_parallel_for:
9099 case OMPD_distribute_parallel_for_simd:
9100 case OMPD_teams_distribute:
9101 case OMPD_teams_distribute_simd:
9102 case OMPD_teams_distribute_parallel_for:
9103 case OMPD_teams_distribute_parallel_for_simd:
9104 case OMPD_target_update:
9105 case OMPD_declare_simd:
9106 case OMPD_declare_target:
9107 case OMPD_end_declare_target:
9108 case OMPD_declare_reduction:
9109 case OMPD_declare_mapper:
9111 case OMPD_taskloop_simd:
9114 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
9119 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
9120 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
9124 E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName);
9129 if (
const auto *L = dyn_cast<LambdaExpr>(S))
9146 if (
const auto *FD = dyn_cast<FunctionDecl>(VD))
9150 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9164 StringRef ParentName =
9169 StringRef ParentName =
9177 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9179 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9180 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9192 "Expected constant variable.");
9194 llvm::Constant *Addr;
9195 llvm::GlobalValue::LinkageTypes
Linkage;
9204 llvm::raw_svector_ostream OS(Buffer);
9205 OS <<
"__omp_offloading_firstprivate_" << llvm::format(
"_%x", DeviceID)
9206 << llvm::format(
"_%x_", FileID) << VD->
getName() <<
"_l" <<
Line;
9213 cast<llvm::GlobalValue>(Addr)->setLinkage(Linkage);
9217 VarName, Addr, VarSize,
9223 llvm::Constant *Addr) {
9225 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9239 llvm::GlobalValue::LinkageTypes
Linkage;
9241 if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9247 assert(!VarSize.
isZero() &&
"Expected non-zero size of the variable");
9254 std::string RefName =
getName({VarName,
"ref"});
9256 llvm::Constant *AddrRef =
9258 auto *GVAddrRef = cast<llvm::GlobalVariable>(AddrRef);
9259 GVAddrRef->setConstant(
true);
9261 GVAddrRef->setInitializer(Addr);
9266 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
9267 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9269 "Declare target attribute must link or to with unified memory.");
9270 if (*Res == OMPDeclareTargetDeclAttr::MT_Link)
9276 VarName = Addr->getName();
9283 Linkage = llvm::GlobalValue::WeakAnyLinkage;
9287 VarName, Addr, VarSize, Flags, Linkage);
9291 if (isa<FunctionDecl>(GD.
getDecl()) ||
9292 isa<OMPDeclareReductionDecl>(GD.
getDecl()))
9301 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9304 if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9308 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
9309 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9311 "Expected link clause or to clause with unified memory.");
9320 " Expected target-based directive.");
9326 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
9335 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
9337 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
9338 switch(A->getAllocatorType()) {
9339 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
9341 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
9342 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
9343 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
9344 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
9345 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
9346 case OMPAllocateDeclAttr::OMPConstMemAlloc:
9347 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
9350 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
9351 llvm_unreachable(
"Expected predefined allocator for the variables with the " 9380 const auto *D = cast<FunctionDecl>(GD.
getDecl());
9383 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
9385 if (
auto *F = dyn_cast_or_null<llvm::Function>(CGM.
GetGlobalValue(Name)))
9386 return !F->isDeclaration();
9408 llvm::Function *RequiresRegFn;
9413 std::string ReqName =
getName({
"omp_offloading",
"requires_reg"});
9426 "Target or declare target region expected.");
9428 Flags = OMP_REQ_UNIFIED_SHARED_MEMORY;
9430 llvm::ConstantInt::get(CGM.
Int64Ty, Flags));
9433 return RequiresRegFn;
9450 llvm::Function *OutlinedFn,
9461 CGF.
Builder.getInt32(CapturedVars.size()),
9464 RealArgs.append(std::begin(Args), std::end(Args));
9465 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
9472 const Expr *NumTeams,
9473 const Expr *ThreadLimit,
9512 auto &&BeginThenGen = [
this, &D, Device, &Info,
9521 MappableExprsHandler MCHandler(D, CGF);
9522 MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
9532 SizesArrayArg, MapTypesArrayArg, Info);
9537 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9540 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9544 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
9547 DeviceID, PointerNum, BasePointersArrayArg,
9548 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
9554 if (!Info.CaptureDeviceAddrMap.empty())
9561 assert(Info.isValid() &&
"Invalid data environment closing arguments.");
9568 SizesArrayArg, MapTypesArrayArg, Info);
9573 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9576 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9580 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
9583 DeviceID, PointerNum, BasePointersArrayArg,
9584 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
9592 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](
CodeGenFunction &CGF,
9594 if (!Info.CaptureDeviceAddrMap.empty()) {
9595 CodeGen.setAction(NoPrivAction);
9613 if (Info.CaptureDeviceAddrMap.empty()) {
9614 CodeGen.setAction(NoPrivAction);
9628 const Expr *Device) {
9632 assert((isa<OMPTargetEnterDataDirective>(D) ||
9633 isa<OMPTargetExitDataDirective>(D) ||
9634 isa<OMPTargetUpdateDirective>(D)) &&
9635 "Expecting either target enter, exit data, or update directives.");
9640 auto &&ThenGen = [
this, &D, Device, &InputInfo,
9645 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9648 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9652 llvm::Constant *PointerNum =
9653 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
9657 InputInfo.BasePointersArray.getPointer(),
9658 InputInfo.PointersArray.getPointer(),
9659 InputInfo.SizesArray.getPointer(),
9667 case OMPD_target_enter_data:
9671 case OMPD_target_exit_data:
9675 case OMPD_target_update:
9681 case OMPD_parallel_for:
9682 case OMPD_parallel_sections:
9684 case OMPD_parallel_for_simd:
9686 case OMPD_cancellation_point:
9688 case OMPD_threadprivate:
9697 case OMPD_taskyield:
9700 case OMPD_taskgroup:
9704 case OMPD_target_data:
9705 case OMPD_distribute:
9706 case OMPD_distribute_simd:
9707 case OMPD_distribute_parallel_for:
9708 case OMPD_distribute_parallel_for_simd:
9709 case OMPD_teams_distribute:
9710 case OMPD_teams_distribute_simd:
9711 case OMPD_teams_distribute_parallel_for:
9712 case OMPD_teams_distribute_parallel_for_simd:
9713 case OMPD_declare_simd:
9714 case OMPD_declare_target:
9715 case OMPD_end_declare_target:
9716 case OMPD_declare_reduction:
9717 case OMPD_declare_mapper:
9719 case OMPD_taskloop_simd:
9721 case OMPD_target_simd:
9722 case OMPD_target_teams_distribute:
9723 case OMPD_target_teams_distribute_simd:
9724 case OMPD_target_teams_distribute_parallel_for:
9725 case OMPD_target_teams_distribute_parallel_for_simd:
9726 case OMPD_target_teams:
9727 case OMPD_target_parallel:
9728 case OMPD_target_parallel_for:
9729 case OMPD_target_parallel_for_simd:
9732 llvm_unreachable(
"Unexpected standalone target data directive.");
9738 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray](
9747 MappableExprsHandler MEHandler(D, CGF);
9748 MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
9757 InputInfo.BasePointersArray =
9759 InputInfo.PointersArray =
9761 InputInfo.SizesArray =
9765 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
9783 struct ParamAttrTy {
9785 llvm::APSInt StrideOrArg;
9786 llvm::APSInt Alignment;
9791 ArrayRef<ParamAttrTy> ParamAttrs) {
9820 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
9821 if (ParamAttrs[Offset].Kind == Vector)
9826 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
9827 if (ParamAttrs[I + Offset].Kind == Vector) {
9844 const llvm::APSInt &VLENVal,
9845 ArrayRef<ParamAttrTy> ParamAttrs,
9846 OMPDeclareSimdDeclAttr::BranchStateTy
State) {
9849 unsigned VecRegSize;
9851 ISADataTy ISAData[] = {
9867 case OMPDeclareSimdDeclAttr::BS_Undefined:
9868 Masked.push_back(
'N');
9869 Masked.push_back(
'M');
9871 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
9872 Masked.push_back(
'N');
9874 case OMPDeclareSimdDeclAttr::BS_Inbranch:
9875 Masked.push_back(
'M');
9878 for (
char Mask : Masked) {
9879 for (
const ISADataTy &Data : ISAData) {
9881 llvm::raw_svector_ostream Out(Buffer);
9882 Out <<
"_ZGV" << Data.ISA << Mask;
9885 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
9886 Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts);
9890 for (
const ParamAttrTy &ParamAttr : ParamAttrs) {
9891 switch (ParamAttr.Kind){
9892 case LinearWithVarStride:
9893 Out <<
's' << ParamAttr.StrideOrArg;
9897 if (!!ParamAttr.StrideOrArg)
9898 Out << ParamAttr.StrideOrArg;
9907 if (!!ParamAttr.Alignment)
9908 Out <<
'a' << ParamAttr.Alignment;
9910 Out <<
'_' << Fn->getName();
9911 Fn->addFnAttr(Out.str());
9933 if (Kind == ParamKindTy::Uniform)
9936 if (Kind == ParamKindTy::Linear)
9941 if (Kind == ParamKindTy::LinearWithVarStride)
9953 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
9988 static std::tuple<unsigned, unsigned, bool>
9994 bool OutputBecomesInput =
false;
9998 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector, C));
10000 OutputBecomesInput =
true;
10002 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10004 Sizes.push_back(
getAArch64LS(QT, ParamAttrs[I].Kind, C));
10007 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
10010 assert(std::all_of(Sizes.begin(), Sizes.end(),
10011 [](
unsigned Size) {
10012 return Size == 8 || Size == 16 || Size == 32 ||
10013 Size == 64 || Size == 128;
10017 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10018 *std::max_element(std::begin(Sizes), std::end(Sizes)),
10019 OutputBecomesInput);
10027 llvm::raw_svector_ostream Out(Buffer);
10028 for (
const auto &ParamAttr : ParamAttrs) {
10029 switch (ParamAttr.Kind) {
10030 case LinearWithVarStride:
10031 Out <<
"ls" << ParamAttr.StrideOrArg;
10037 if (!!ParamAttr.StrideOrArg && ParamAttr.StrideOrArg != 1)
10038 Out << ParamAttr.StrideOrArg;
10048 if (!!ParamAttr.Alignment)
10049 Out <<
'a' << ParamAttr.Alignment;
10058 template <
typename T>
10060 char ISA, StringRef ParSeq,
10061 StringRef MangledName,
bool OutputBecomesInput,
10062 llvm::Function *Fn) {
10064 llvm::raw_svector_ostream Out(Buffer);
10065 Out << Prefix << ISA << LMask << VLEN;
10066 if (OutputBecomesInput)
10068 Out << ParSeq <<
"_" << MangledName;
10069 Fn->addFnAttr(Out.str());
10075 StringRef Prefix,
char ISA,
10076 StringRef ParSeq, StringRef MangledName,
10077 bool OutputBecomesInput,
10078 llvm::Function *Fn) {
10082 OutputBecomesInput, Fn);
10084 OutputBecomesInput, Fn);
10088 OutputBecomesInput, Fn);
10090 OutputBecomesInput, Fn);
10094 OutputBecomesInput, Fn);
10096 OutputBecomesInput, Fn);
10101 OutputBecomesInput, Fn);
10104 llvm_unreachable(
"Scalar type is too wide.");
10111 ArrayRef<ParamAttrTy> ParamAttrs,
10112 OMPDeclareSimdDeclAttr::BranchStateTy
State, StringRef MangledName,
10113 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
10116 const auto Data =
getNDSWDS(FD, ParamAttrs);
10117 const unsigned NDS = std::get<0>(Data);
10118 const unsigned WDS = std::get<1>(Data);
10119 const bool OutputBecomesInput = std::get<2>(Data);
10123 if (UserVLEN == 1) {
10126 "The clause simdlen(1) has no effect when targeting aarch64.");
10133 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10136 "power of 2 when targeting Advanced SIMD.");
10143 if (ISA ==
's' && UserVLEN != 0) {
10144 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10147 "lanes in the architectural constraints " 10148 "for SVE (min is 128-bit, max is " 10149 "2048-bit, by steps of 128-bit)");
10157 StringRef Prefix =
"_ZGV";
10163 OutputBecomesInput, Fn);
10165 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10169 case OMPDeclareSimdDeclAttr::BS_Undefined:
10171 OutputBecomesInput, Fn);
10173 OutputBecomesInput, Fn);
10175 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10177 OutputBecomesInput, Fn);
10179 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10181 OutputBecomesInput, Fn);
10191 OutputBecomesInput, Fn);
10193 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10198 case OMPDeclareSimdDeclAttr::BS_Undefined:
10200 OutputBecomesInput, Fn);
10202 OutputBecomesInput, Fn);
10204 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10206 OutputBecomesInput, Fn);
10208 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10210 OutputBecomesInput, Fn);
10218 llvm::Function *Fn) {
10222 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10223 if (isa<CXXMethodDecl>(FD))
10224 ParamPositions.try_emplace(FD, 0);
10225 unsigned ParamPos = ParamPositions.size();
10227 ParamPositions.try_emplace(
P->getCanonicalDecl(), ParamPos);
10234 for (
const Expr *E :
Attr->uniforms()) {
10237 if (isa<CXXThisExpr>(E)) {
10238 Pos = ParamPositions[FD];
10240 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10242 Pos = ParamPositions[PVD];
10244 ParamAttrs[Pos].Kind = Uniform;
10247 auto NI =
Attr->alignments_begin();
10248 for (
const Expr *E :
Attr->aligneds()) {
10252 if (isa<CXXThisExpr>(E)) {
10253 Pos = ParamPositions[FD];
10256 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10258 Pos = ParamPositions[PVD];
10259 ParmTy = PVD->getType();
10261 ParamAttrs[Pos].Alignment =
10263 ? (*NI)->EvaluateKnownConstInt(C)
10264 : llvm::APSInt::getUnsigned(
10270 auto SI =
Attr->steps_begin();
10271 auto MI =
Attr->modifiers_begin();
10272 for (
const Expr *E :
Attr->linears()) {
10275 if (isa<CXXThisExpr>(E)) {
10276 Pos = ParamPositions[FD];
10278 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10280 Pos = ParamPositions[PVD];
10282 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
10283 ParamAttr.Kind = Linear;
10287 if (
const auto *DRE =
10288 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
10289 if (
const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
10290 ParamAttr.Kind = LinearWithVarStride;
10291 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
10292 ParamPositions[StridePVD->getCanonicalDecl()]);
10296 ParamAttr.StrideOrArg = Result.
Val.
getInt();
10302 llvm::APSInt VLENVal;
10304 const Expr *VLENExpr =
Attr->getSimdlen();
10309 OMPDeclareSimdDeclAttr::BranchStateTy
State =
Attr->getBranchState();
10310 if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
10311 CGM.getTriple().getArch() == llvm::Triple::x86_64) {
10313 }
else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
10314 unsigned VLEN = VLENVal.getExtValue();
10315 StringRef MangledName = Fn->getName();
10316 if (CGM.getTarget().hasFeature(
"sve"))
10318 MangledName,
's', 128, Fn, ExprLoc);
10319 if (CGM.getTarget().hasFeature(
"neon"))
10321 MangledName,
'n', 128, Fn, ExprLoc);
10332 static const int DoacrossFinArgs = 2;
10335 llvm::FunctionCallee RTLFn;
10339 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
10342 assert(CallArgs.size() == DoacrossFinArgs);
10343 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
10378 llvm::APInt Size(32, NumIterations.size());
10384 enum { LowerFD = 0, UpperFD, StrideFD };
10386 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
10391 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
10399 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
10409 llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
10414 llvm::FunctionCallee RTLFn =
10417 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
10419 llvm::FunctionCallee FiniRTLFn =
10422 llvm::makeArrayRef(FiniArgs));
10428 CGM.getContext().getIntTypeForBitwidth(64, 1);
10430 QualType ArrayTy = CGM.getContext().getConstantArrayType(
10433 for (
unsigned I = 0, E = C->
getNumLoops(); I < E; ++I) {
10435 assert(CounterVal);
10446 llvm::FunctionCallee RTLFn;
10457 llvm::FunctionCallee Callee,
10459 assert(Loc.
isValid() &&
"Outlined function call location must be valid.");
10462 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
10463 if (Fn->doesNotThrow()) {
10474 emitCall(CGF, Loc, OutlinedFn, Args);
10478 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
10479 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
10485 const VarDecl *TargetParam)
const {
10493 static const int CleanupArgs = 3;
10496 llvm::FunctionCallee RTLFn;
10500 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
10503 assert(CallArgs.size() == CleanupArgs &&
10504 "Size of arguments does not match.");
10505 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
10520 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
10522 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
10524 if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
10525 !AA->getAllocator())
10528 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
10532 Size = CGF.
Builder.CreateNUWAdd(
10534 Size = CGF.
Builder.CreateUDiv(Size, CGM.getSize(Align));
10535 Size = CGF.
Builder.CreateNUWMul(Size, CGM.getSize(Align));
10538 Size = CGM.getSize(Sz.
alignTo(Align));
10541 assert(AA->getAllocator() &&
10542 "Expected allocator expression for non-default allocator.");
10546 if (Allocator->getType()->isIntegerTy())
10547 Allocator = CGF.
Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
10548 else if (Allocator->getType()->isPointerTy())
10551 llvm::Value *Args[] = {ThreadID, Size, Allocator};
10555 CVD->
getName() +
".void.addr");
10556 llvm::Value *FiniArgs[OMPAllocateCleanupTy::CleanupArgs] = {ThreadID, Addr,
10561 llvm::makeArrayRef(FiniArgs));
10572 llvm_unreachable(
"Not supported in SIMD-only mode");
10578 llvm_unreachable(
"Not supported in SIMD-only mode");
10585 bool Tied,
unsigned &NumberOfParts) {
10586 llvm_unreachable(
"Not supported in SIMD-only mode");
10591 llvm::Function *OutlinedFn,
10593 const Expr *IfCond) {
10594 llvm_unreachable(
"Not supported in SIMD-only mode");
10600 const Expr *Hint) {
10601 llvm_unreachable(
"Not supported in SIMD-only mode");
10607 llvm_unreachable(
"Not supported in SIMD-only mode");
10612 llvm_unreachable(
"Not supported in SIMD-only mode");
10618 llvm_unreachable(
"Not supported in SIMD-only mode");
10626 llvm_unreachable(
"Not supported in SIMD-only mode");
10633 llvm_unreachable(
"Not supported in SIMD-only mode");
10640 bool ForceSimpleCall) {
10641 llvm_unreachable(
"Not supported in SIMD-only mode");
10648 llvm_unreachable(
"Not supported in SIMD-only mode");
10654 llvm_unreachable(
"Not supported in SIMD-only mode");
10660 llvm_unreachable(
"Not supported in SIMD-only mode");
10667 llvm_unreachable(
"Not supported in SIMD-only mode");
10673 llvm_unreachable(
"Not supported in SIMD-only mode");
10678 unsigned IVSize,
bool IVSigned,
10681 llvm_unreachable(
"Not supported in SIMD-only mode");
10687 llvm_unreachable(
"Not supported in SIMD-only mode");
10693 llvm_unreachable(
"Not supported in SIMD-only mode");
10700 llvm_unreachable(
"Not supported in SIMD-only mode");
10706 llvm_unreachable(
"Not supported in SIMD-only mode");
10711 llvm_unreachable(
"Not supported in SIMD-only mode");
10717 llvm_unreachable(
"Not supported in SIMD-only mode");
10722 llvm::Function *TaskFunction,
10724 const Expr *IfCond,
10726 llvm_unreachable(
"Not supported in SIMD-only mode");
10733 llvm_unreachable(
"Not supported in SIMD-only mode");
10740 assert(Options.
SimpleReduction &&
"Only simple reduction is expected.");
10742 ReductionOps, Options);
10748 llvm_unreachable(
"Not supported in SIMD-only mode");
10755 llvm_unreachable(
"Not supported in SIMD-only mode");
10762 llvm_unreachable(
"Not supported in SIMD-only mode");
10767 llvm_unreachable(
"Not supported in SIMD-only mode");
10773 llvm_unreachable(
"Not supported in SIMD-only mode");
10779 llvm_unreachable(
"Not supported in SIMD-only mode");
10784 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
10786 llvm_unreachable(
"Not supported in SIMD-only mode");
10791 llvm::Function *OutlinedFn,
10793 const Expr *IfCond,
10794 const Expr *Device) {
10795 llvm_unreachable(
"Not supported in SIMD-only mode");
10799 llvm_unreachable(
"Not supported in SIMD-only mode");
10803 llvm_unreachable(
"Not supported in SIMD-only mode");
10817 llvm::Function *OutlinedFn,
10819 llvm_unreachable(
"Not supported in SIMD-only mode");
10823 const Expr *NumTeams,
10824 const Expr *ThreadLimit,
10826 llvm_unreachable(
"Not supported in SIMD-only mode");
10832 llvm_unreachable(
"Not supported in SIMD-only mode");
10837 const Expr *Device) {
10838 llvm_unreachable(
"Not supported in SIMD-only mode");
10844 llvm_unreachable(
"Not supported in SIMD-only mode");
10849 llvm_unreachable(
"Not supported in SIMD-only mode");
10854 const VarDecl *NativeParam)
const {
10855 llvm_unreachable(
"Not supported in SIMD-only mode");
10861 const VarDecl *TargetParam)
const {
10862 llvm_unreachable(
"Not supported in SIMD-only mode");
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
llvm::PointerType * Int8PtrPtrTy
RecordDecl * buildImplicitRecord(StringRef Name, RecordDecl::TagKind TK=TTK_Struct) const
Create a new implicit TU-level CXXRecordDecl or RecordDecl declaration.
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
void pushTerminate()
Push a terminate handler on the stack.
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, llvm::Type *BaseLVType, CharUnits BaseLVAlignment, llvm::Value *Addr)
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
llvm::IntegerType * IntTy
int
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
External linkage, which indicates that the entity can be referred to from other translation units...
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
Expr * getUpperBoundVariable() const
Other implicit parameter.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param Data Additional data for task generation like final list of privates etc *TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
QualType TgtDeviceImageQTy
struct __tgt_device_image{ void *ImageStart; // Pointer to the target code start. ...
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
PointerType - C99 6.7.5.1 - Pointer Declarators.
Scheduling data for loop-based OpenMP directives.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
A (possibly-)qualified type.
CodeGenTypes & getTypes()
llvm::Function * emitReductionFunction(SourceLocation Loc, llvm::Type *ArgsType, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, ArrayRef< const Expr *> ReductionOps)
Emits reduction function.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
const CodeGenOptions & getCodeGenOpts() const
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
CGRecordLayout - This class handles struct and union layout info while lowering AST types to LLVM typ...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
const RecordDecl * KmpTaskTQTyRD
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
void actOnDeviceGlobalVarEntriesInfo(const OffloadDeviceGlobalVarEntryInfoActTy &Action)
The standard implementation of ConstantInitBuilder used in Clang.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D...
Stmt - This represents one statement.
Expr * getLowerBoundVariable() const
OpenMPOffloadingRequiresDirFlags
Values for bit flags for marking which requires clauses have been used.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
Expr * getLoopData(unsigned NumLoop)
Get the loop data.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
CharUnits getAlignOfGlobalVarInChars(QualType T) const
Return the alignment in characters that should be given to a global variable with type T...
QualType getTgtBinaryDescriptorQTy()
Returns __tgt_bin_desc type.
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator, StringRef Separator)
Constructor allowing to redefine the name separator for the variables.
bool isRecordType() const
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter...
SmallVector< const Expr *, 4 > LastprivateCopies
Decl - This represents one declaration (or definition), e.g.
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
static llvm::Value * getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, llvm::Value *DefaultThreadLimitVal)
static llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the number of teams for a target directive.
Entry is a target region.
CharUnits getPointerSize() const
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
const RecordType * getAsStructureType() const
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Function * emitTeamsOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
This represents 'if' clause in the '#pragma omp ...' directive.
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, ArrayRef< const Expr *> PrivateVars, ArrayRef< const Expr *> FirstprivateVars, ArrayRef< const Expr *> LastprivateVars, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables...
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
Call to void __kmpc_threadprivate_register( ident_t *, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
ReductionCodeGen(ArrayRef< const Expr *> Shareds, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> ReductionOps)
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
The base class of the type hierarchy.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, ArrayRef< const Expr *> ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
llvm::Value * PointersArray
The array of section pointers passed to the runtime library.
virtual void completeDefinition()
Note that the definition of this type is now complete.
bool isZero() const
isZero - Test whether the quantity equals zero.
QualType withConst() const
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
const TargetInfo & getTargetInfo() const
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
capture_const_range captures() const
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Expr * getCondition() const
Returns condition.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference...
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to...
virtual void checkArchForUnifiedAddressing(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing...
Describes the capture of a variable or of this, or of a C++1y init-capture.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and and emit all target regions found along the way.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
Represents a C++ constructor within a class.
static llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the number of threads for a target directive.
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS...
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
OpenMPSchedType
Schedule types for 'omp for' loops (these enumerators are taken from the enum sched_type in kmp...
bool isTrivialType(const ASTContext &Context) const
Return true if this is a trivial type per (C++0x [basic.types]p9)
SmallVector< const Expr *, 4 > ReductionCopies
SourceLocation getEndLoc() const
Returns ending location of directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
Objects with "hidden" visibility are not seen by the dynamic linker.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
QualType getReturnType() const
This represents 'num_threads' clause in the '#pragma omp ...' directive.
const T * getAs() const
Member-template getAs<specific type>'.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, const Expr *Device)
Emit the target offloading code associated with D.
Extra information about a function prototype.
bool supportsCOMDAT() const
LangAS
Defines the address space values used by the address space qualifier of QualType. ...
llvm::GlobalVariable * finishAndCreateGlobal(As &&...args)
Given that this builder was created by beginning an array or struct directly on a ConstantInitBuilder...
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
DiagnosticsEngine & getDiags() const
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
llvm::Value * getPointer() const
llvm::Type * ConvertTypeForMem(QualType T)
ConvertTypeForMem - Convert type T into a llvm::Type.
static llvm::GlobalVariable * createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant, ArrayRef< llvm::Constant *> Data, const Twine &Name, As &&... Args)
Represents a parameter to a function.
Linkage
Describes the different kinds of linkage (C++ [basic.link], C99 6.2.2) that an entity may have...
static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, const Expr *Ref)
Generates unique name for artificial threadprivate variables.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata...
llvm::FunctionCallee createForStaticInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned...
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
Struct that keeps all the relevant information that should be kept throughout a 'target data' region...
QualType getTgtOffloadEntryQTy()
Returns __tgt_offload_entry type.
llvm::Function * TaskEntry
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
SmallVector< const Expr *, 4 > PrivateVars
Represents a struct/union/class.
clauselist_range clauselists()
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
Source[4] in Fortran, do not use for C++.
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target...
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
virtual llvm::Function * emitParallelOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
Address getAddress() const
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Call to void *__kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid, void *data...
llvm::IntegerType * Int64Ty
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
SmallVector< const Expr *, 4 > LastprivateVars
SourceLocation getBeginLoc() const LLVM_READONLY
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
Represents a member of a struct/union/class.
CharUnits getAlignment() const
llvm::IntegerType * SizeTy
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3...
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item)...
LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, bool IsLowerBound=true)
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
static void emitOffloadingArrays(CodeGenFunction &CGF, MappableExprsHandler::MapBaseValuesArrayTy &BasePointers, MappableExprsHandler::MapValuesArrayTy &Pointers, MappableExprsHandler::MapValuesArrayTy &Sizes, MappableExprsHandler::MapFlagsArrayTy &MapTypes, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the arrays used to pass the captures and map information to the offloading runtime library...
void startDefinition()
Starts the definition of this tag declaration.
bool isReferenceType() const
This represents clause 'map' in the '#pragma omp ...' directives.
InitKind getInitializerKind() const
Get initializer kind.
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
OMPTargetGlobalVarEntryKind
Kind of the global variable entry..
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable. ...
This represents clause 'to' in the '#pragma omp ...' directives.
static CharUnits Zero()
Zero - Construct a CharUnits quantity of zero.
clang::CharUnits operator*(clang::CharUnits::QuantityType Scale, const clang::CharUnits &CU)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
OpenMPDirectiveKind getDirectiveKind() const
__DEVICE__ int max(int __a, int __b)
SourceLocation getBeginLoc() const LLVM_READONLY
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
ArrayRef< ParmVarDecl * > parameters() const
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
Mark the entry as a to declare target.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
void emitProcBindClause(CodeGenFunction &CGF, OpenMPProcBindClauseKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr *> NumIterations)
Emit initialization for doacross loop nesting support.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
QualType TgtOffloadEntryQTy
Type struct __tgt_offload_entry{ void *addr; // Pointer to the offload entry info.
static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM, const RecordDecl *RD, const CGRecordLayout &RL, ArrayRef< llvm::Constant *> Data)
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
llvm::Function * emitRegistrationFunction() override
Creates the offloading descriptor in the event any target region was emitted in the current module an...
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 3.1.1 of the AAVFABI.
OpenMPScheduleClauseKind Schedule
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
APValue Val
Val - This is the value the expression can be folded to.
bool requiresLandingPad() const
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info)
Emit the target data mapping code associated with D.
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
RAII for correct setting/restoring of CapturedStmtInfo.
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
CharUnits getAlignment() const
Return the alignment of this pointer.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits code for a taskyield directive.
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
llvm::PointerType * VoidPtrTy
bool empty() const
Return true if a there are no entries defined.
String describing the source location.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
void actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action)
OpenMPScheduleClauseModifier M2
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind...
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, const Expr *Device) override
Emit the target offloading code associated with D.
SmallVector< const Expr *, 4 > PrivateCopies
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned NumberOfPtrs
The total number of pointers passed to the runtime library.
void operator()(CodeGenFunction &CGF) const
bool isConstexpr() const
Whether this variable is (C++11) constexpr.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
bool isLambda() const
Determine whether this class describes a lambda function object.
llvm::StringSet AlreadyEmittedTargetFunctions
List of the emitted functions.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Expr * getSizeExpr() const
Scope - A scope is a transient data structure that is used while parsing the program.
llvm::PointerType * VoidPtrPtrTy
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
field_iterator field_begin() const
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
ArrayRef< MappableComponent > MappableExprComponentListRef
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void addCompilerUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.compiler.used metadata.
SmallVector< const Expr *, 4 > FirstprivateCopies
OpenMPDependClauseKind getDependencyKind() const
Get dependency type.
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static int addMonoNonMonoModifier(OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
DiagnosticsEngine & getDiagnostics() const
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
static void emitOffloadingArraysArgument(CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg, llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the arguments to be passed to the runtime library based on the arrays of pointers, sizes and map types.
Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0)
Emits object of ident_t type with info for source location.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CharUnits getPointerAlign() const
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
SmallVector< const Expr *, 4 > ReductionOps
This represents clause 'is_device_ptr' in the '#pragma omp ...' directives.
SmallVector< const Expr *, 4 > ReductionVars
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop...
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
This represents clause 'from' in the '#pragma omp ...' directives.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
OpenMP 4.0 [2.4, Array Sections].
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
ASTContext & getContext() const
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
Describes the capture of either a variable, or 'this', or variable-length array type.
const CodeGen::CGBlockInfo * BlockInfo
ArrayBuilder beginArray(llvm::Type *eltTy=nullptr)
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter...
CGBlockInfo - Information to generate a block literal.
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
LValueBaseInfo getBaseInfo() const
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
Class intended to support codegen of all kind of the reduction clauses.
llvm::FunctionCallee createRuntimeFunction(unsigned Function)
Returns specified OpenMP runtime function.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
unsigned getNumLoops() const
Get number of loops associated with the clause.
This represents '#pragma omp requires...' directive.
virtual void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags, llvm::GlobalValue::LinkageTypes Linkage)
Creates offloading entry for the provided entry ID ID, address Addr, size Size, and flags Flags...
llvm::CallingConv::ID getRuntimeCC() const
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
QualType getTgtDeviceImageQTy()
Returns __tgt_device_image type.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
virtual unsigned getDefaultFirstprivateAddressSpace() const
Returns default address space for the constant firstprivates, 0 by default.
KmpTaskTFields
Indexes of fields for type kmp_task_t.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr *> Vars, SourceLocation Loc)
Emit flush of the variables specified in 'omp flush' directive.
static void createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty, ArrayRef< llvm::Constant *> Data, T &Parent)
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emit code for 'taskwait' directive.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
This represents one expression.
Allow any unmodeled side effect.
void loadOffloadInfoMetadata()
Loads all the offload entries information from the host IR metadata.
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
const AnnotatedLine * Line
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
const CGFunctionInfo & arrangeNullaryFunction()
A nullary function is a freestanding function of type 'void ()'.
std::string OMPHostIRFile
Name of the IR file that contains the result of the OpenMP target host code generation.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
const T * castAs() const
Member-template castAs<specific type>.
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
llvm::BasicBlock * getBlock() const
unsigned getLine() const
Return the presumed line number of this location.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD)
Checks if destructor function is required to be generated.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr *> CopyprivateVars, ArrayRef< const Expr *> DestExprs, ArrayRef< const Expr *> SrcExprs, ArrayRef< const Expr *> AssignmentOps) override
Emits a single region.
Represents a C++ destructor within a class.
VariableCaptureKind getCaptureKind() const
Determine the kind of capture.
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements, of a variable length array type, plus that largest non-variably-sized element type.
field_iterator field_end() const
llvm::PointerType * getType() const
Return the type of the pointer value.
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
DeclContext * getDeclContext()
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Mark the entry as a to declare target link.
llvm::FunctionCallee createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr *> Vars, SourceLocation Loc) override
Emit flush of the variables specified in 'omp flush' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
llvm::IntegerType * Int32Ty
llvm::GlobalValue::LinkageTypes getLLVMLinkageVarDefinition(const VarDecl *VD, bool IsConstant)
Returns LLVM linkage for a declarator.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
Provides LLVM's BitmaskEnum facility to enumeration types declared in namespace clang.
SmallVector< const Expr *, 4 > FirstprivateVars
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, ArrayType::ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type...
TBAAAccessInfo getTBAAInfo() const
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr *> CopyprivateVars, ArrayRef< const Expr *> DestExprs, ArrayRef< const Expr *> SrcExprs, ArrayRef< const Expr *> AssignmentOps, SourceLocation Loc)
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
QualType getRecordType(const RecordDecl *Decl) const
Represents an unpacked "presumed" location which can be presented to the user.
void Emit(CodeGenFunction &CGF, Flags) override
Emit the cleanup.
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, llvm::Constant *Addr, llvm::Constant *ID, OMPTargetRegionEntryKind Flags)
Register target region entry.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
Expr * getInitPriv()
Get Priv variable of the initializer.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
const Qualifiers & getQuals() const
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
const LangOptions & getLangOpts() const
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
ASTContext & getContext() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
OpenMPProcBindClauseKind
OpenMP attributes for 'proc_bind' clause.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
GlobalDecl - represents a global declaration.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
void finishAndAddTo(AggregateBuilderBase &parent)
Given that this builder was created by beginning an array or struct component on the given parent bui...
CanQualType getCanonicalTypeUnqualified() const
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
QualType TgtBinaryDescriptorQTy
struct __tgt_bin_desc{ int32_t NumDevices; // Number of devices supported.
The l-value was considered opaque, so the alignment was determined from a type.
RecordDecl * getDecl() const
const char * getFilename() const
Return the presumed filename of this location.
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ...
virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
Expr * getStrideVariable() const
This represents 'num_teams' clause in the '#pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
CanProxy< U > castAs() const
This captures a statement into a function.
QualType getCanonicalType() const
decl_type * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
llvm::Function * emitRequiresDirectiveRegFun()
Creates and returns a registration function for when at least one requires directives was used in the...
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
unsigned getColumn() const
Return the presumed column number of this location.
static with chunk adjustment (e.g., simd)
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup...
void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
Encodes a location in the source.
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
llvm::FunctionCallee createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned...
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
llvm::Value * MapTypesArray
The array of map types passed to the runtime library.
This represents '#pragma omp declare reduction ...' directive.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr *> CopyprivateVars, ArrayRef< const Expr *> DestExprs, ArrayRef< const Expr *> SrcExprs, ArrayRef< const Expr *> AssignmentOps)
Emits a single region.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars, const Expr *IfCond) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, const llvm::Twine &Name="")
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams...
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
This is a basic class for representing single OpenMP executable directive.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
Lower bound for 'ordered' versions.
ASTContext & getASTContext() const LLVM_READONLY
const Decl * getDecl() const
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
OpenMPDirectiveKind
OpenMP directives.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr *> NumIterations) override
Emit initialization for doacross loop nesting support.
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ...
Set if the nonmonotonic schedule modifier was present.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
Target region entries info.
const ParmVarDecl * getParamDecl(unsigned i) const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
Device global variable entries info.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars, const Expr *IfCond)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
This is a basic class for representing single OpenMP clause.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
bool isAnyPointerType() const
This declaration is only a declaration.
unsigned size() const
Return number of entries defined so far.
virtual void Enter(CodeGenFunction &CGF)
llvm::FunctionCallee createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned...
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after...
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
virtual void emitProcBindClause(CodeGenFunction &CGF, OpenMPProcBindClauseKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
llvm::Value * NewTaskNewTaskTTy
bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum) const
Return true if a target region entry with the provided information exists.
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value *> Args=llvm::None) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
Expr * getInitOrig()
Get Orig variable of the initializer.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
void registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr, CharUnits VarSize, OMPTargetGlobalVarEntryKind Flags, llvm::GlobalValue::LinkageTypes Linkage)
Register device global variable entry.
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character...
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void addUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.used metadata.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
QualType withRestrict() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
StructBuilder beginStruct(llvm::StructType *ty=nullptr)
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::GlobalValue * GetGlobalValue(StringRef Ref)
Dataflow Directional Tag Classes.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
This represents 'device' clause in the '#pragma omp ...' directive.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
EvalResult is a struct with detailed info about an evaluated expression.
static std::string getName(const CallEvent &Call)
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
OpenMPScheduleClauseModifier M1
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getInit() const
llvm::Constant * getPointer() const
void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, unsigned Order)
Initialize target region entry.
llvm::Function * createOffloadingBinaryDescriptorRegistration()
Creates and registers offloading binary descriptor for the current compilation unit.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::IntegerType * IntPtrTy
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
virtual void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, llvm::FunctionCallee Dtor, llvm::Constant *Addr)=0
Emit code to force the execution of a destructor during global teardown.
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::Module & getModule() const
Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...
Not really used in Fortran any more.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
llvm::StructType * ConvertRecordDeclType(const RecordDecl *TD)
ConvertRecordDeclType - Lay out a tagged decl type like struct or union.
virtual bool isDefaultLocationConstant() const
Check if the default location must be constant.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
OffloadEntriesInfoManagerTy OffloadEntriesInfoManager
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
Class that represents a component of a mappable expression.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
void EmitAutoVarCleanups(const AutoVarEmission &emission)
API for captured statement code generation.
virtual bool emitDeclareTargetVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Addr, bool PerformInit)
Emit a code for initialization of declare target variable.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
virtual StringRef getOutlinedHelperName() const
Get the function name of an outlined region.
static bool classof(const OMPClause *T)
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
ArraySubscriptExpr - [C99 6.5.2.1] Array Subscripting.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
This file defines OpenMP AST classes for executable directives and clauses.
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
CodeGenTypes & getTypes() const
StructBuilder beginStruct(llvm::StructType *structTy=nullptr)
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
CleanupTy(PrePostActionTy *Action)
llvm::Type * getElementType() const
Return the type of the values stored in this address.
void initializeDeviceGlobalVarEntryInfo(StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order)
Initialize device global variable entry.
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Value *Ptr, ArrayRef< llvm::Value *> IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc, unsigned &DeviceID, unsigned &FileID, unsigned &LineNum)
Obtain information that uniquely identifies a target entry.
llvm::PointerType * Int8PtrTy
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
OpenMPLocationFlags
Values for bit flags used in the ident_t to describe the fields.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Expr * getNumIterations() const
llvm::StringRef getName() const
Return the IR name of the pointer value.
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
Base for LValueReferenceType and RValueReferenceType.
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
Entity that registers the offloading constants that were emitted so far.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value *> Args=llvm::None) const
Emits Callee function call with arguments Args with location Loc.
StringRef getMangledName(GlobalDecl GD)
llvm::Constant * getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace=0)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false)
Create or return a runtime function declaration with the specified type and name. ...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void addDecl(Decl *D)
Add the declaration D into this context.
FieldDecl * LambdaThisCaptureField
llvm::Constant * registerTargetFirstprivateCopy(CodeGenFunction &CGF, const VarDecl *VD)
Registers provided target firstprivate variable as global on the target.
void getCaptureFields(llvm::DenseMap< const VarDecl *, FieldDecl *> &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
Mark the entry as target region.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
SourceManager & getSourceManager()
virtual llvm::Function * emitRegistrationFunction()
Creates the offloading descriptor in the event any target region was emitted in the current module an...
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
Mark the entry as a global destructor.
Address LoadCXXThisAddress()
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
Lower bound for default (unordered) versions.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams...
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
TranslationUnitDecl * getTranslationUnitDecl() const
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, ArrayRef< const Expr *> ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
This represents 'nowait' clause in the '#pragma omp ...' directive.
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls. ...
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
Represents a C++ struct/union/class.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
llvm::Function * CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false)
llvm::Value * BasePointersArray
The array of base pointer passed to the runtime library.
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
~DisableAutoDeclareTargetRAII()
OpenMPOffloadingReservedDeviceIDs
llvm::Type * ConvertType(QualType T)
bool isTLSSupported() const
Whether the target supports thread-local storage.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
Privates[]
Gets the list of initial values for linear variables.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
virtual llvm::Function * emitTeamsOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
A helper class of ConstantInitBuilder, used for building constant struct initializers.
void popTerminate()
Pops a terminate handler off the stack.
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
Entry is a declare target variable.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
llvm::Function * emitParallelOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
CGCXXABI & getCXXABI() const
Expr * getCombinerIn()
Get In variable of the combiner.
const VariableArrayType * getAsVariableArrayType(QualType T) const
Mark the entry as a global constructor.
decl_type * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
A reference to a declared variable, function, enum, etc.
QualType getIntPtrType() const
Return a type compatible with "intptr_t" (C99 7.18.1.4), as defined by the target.
static RValue get(llvm::Value *V)
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
bool isPointerType() const
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
ParamKindTy
Kind of parameter in a function with 'declare simd' directive.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop...
An l-value expression is a reference to an object with independent storage.
bool isFloatingType() const
static RValue getAggregate(Address addr, bool isVolatile=false)
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
LValue - This represents an lvalue references.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
Information for lazily generating a cleanup.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
Represents a C array with a specified size that is not an integer-constant-expression.
bool isConstant(const ASTContext &Ctx) const
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
const LangOptions & getLangOpts() const
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::StringSet DeclareTargetWithDefinition
Set of declare target variables with the generated initializer.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
Expr * getCombinerOut()
Get Out variable of the combiner.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
llvm::Value * getPointer() const
base_class_range vbases()
This class handles loading and caching of source files into memory.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
A helper class of ConstantInitBuilder, used for building constant array initializers.
void EmitGlobal(GlobalDecl D)
Emit code for a single global function or var decl.
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
Attr - This represents one attribute.
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
bool isExternallyVisible() const
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object...
llvm::Value * SizesArray
The array of sizes passed to the runtime library.
virtual void emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *Device, const llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> &SizeEmitter)
Emit code that pushes the trip count of loops associated with constructs 'target teams distribute' an...
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc)
Emit code for 'taskwait' directive.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.