12 #include "llvm/IR/BasicBlock.h" 13 #include "llvm/IR/CFG.h" 14 #include "llvm/IR/Constants.h" 15 #include "llvm/IR/InstrTypes.h" 16 #include "llvm/IR/Instructions.h" 17 #include "llvm/IR/Metadata.h" 23 LLVMContext &Ctx = Header->getContext();
25 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26 NewLoopProperties.push_back(TempNode.get());
27 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30 LoopID->replaceOperandWith(0, LoopID);
34 MDNode *LoopInfo::createPipeliningMetadata(
const LoopAttributes &Attrs,
36 bool &HasUserTransforms) {
37 LLVMContext &Ctx = Header->getContext();
45 if (Enabled !=
true) {
47 if (Enabled ==
false) {
48 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49 NewLoopProperties.push_back(
50 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.pipeline.disable"),
51 ConstantAsMetadata::get(ConstantInt::get(
52 llvm::Type::getInt1Ty(Ctx), 1))}));
53 LoopProperties = NewLoopProperties;
55 return createLoopPropertiesMetadata(LoopProperties);
59 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60 Args.push_back(TempNode.get());
61 Args.append(LoopProperties.begin(), LoopProperties.end());
65 MDString::get(Ctx,
"llvm.loop.pipeline.initiationinterval"),
66 ConstantAsMetadata::get(ConstantInt::get(
68 Args.push_back(MDNode::get(Ctx, Vals));
73 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74 LoopID->replaceOperandWith(0, LoopID);
75 HasUserTransforms =
true;
82 bool &HasUserTransforms) {
83 LLVMContext &Ctx = Header->getContext();
94 if (Enabled !=
true) {
97 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
103 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
106 FollowupLoopProperties.push_back(
107 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
109 bool FollowupHasTransforms =
false;
110 MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111 FollowupHasTransforms);
114 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115 Args.push_back(TempNode.get());
116 Args.append(LoopProperties.begin(), LoopProperties.end());
120 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.count"),
121 ConstantAsMetadata::get(ConstantInt::get(
123 Args.push_back(MDNode::get(Ctx, Vals));
128 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.enable")};
129 Args.push_back(MDNode::get(Ctx, Vals));
132 if (FollowupHasTransforms)
133 Args.push_back(MDNode::get(
134 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll.followup_all"), Followup}));
136 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137 LoopID->replaceOperandWith(0, LoopID);
138 HasUserTransforms =
true;
145 bool &HasUserTransforms) {
146 LLVMContext &Ctx = Header->getContext();
155 if (Enabled !=
true) {
157 if (Enabled ==
false) {
158 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159 NewLoopProperties.push_back(MDNode::get(
160 Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
161 LoopProperties = NewLoopProperties;
163 return createPartialUnrollMetadata(Attrs, LoopProperties,
168 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169 FollowupLoopProperties.push_back(
170 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
172 bool FollowupHasTransforms =
false;
173 MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174 FollowupHasTransforms);
177 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178 Args.push_back(TempNode.get());
179 Args.append(LoopProperties.begin(), LoopProperties.end());
184 MDString::get(Ctx,
"llvm.loop.unroll_and_jam.count"),
185 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
187 Args.push_back(MDNode::get(Ctx, Vals));
191 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.enable")};
192 Args.push_back(MDNode::get(Ctx, Vals));
195 if (FollowupHasTransforms)
196 Args.push_back(MDNode::get(
197 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_outer"),
200 if (UnrollAndJamInnerFollowup)
201 Args.push_back(MDNode::get(
202 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_inner"),
203 UnrollAndJamInnerFollowup}));
205 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206 LoopID->replaceOperandWith(0, LoopID);
207 HasUserTransforms =
true;
212 LoopInfo::createLoopVectorizeMetadata(
const LoopAttributes &Attrs,
214 bool &HasUserTransforms) {
215 LLVMContext &Ctx = Header->getContext();
224 if (Enabled !=
true) {
226 if (Enabled ==
false) {
227 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228 NewLoopProperties.push_back(
229 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
230 ConstantAsMetadata::get(ConstantInt::get(
231 llvm::Type::getInt1Ty(Ctx), 0))}));
232 LoopProperties = NewLoopProperties;
234 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
239 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
242 FollowupLoopProperties.push_back(
243 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
245 bool FollowupHasTransforms =
false;
246 MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247 FollowupHasTransforms);
250 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
251 Args.push_back(TempNode.get());
252 Args.append(LoopProperties.begin(), LoopProperties.end());
257 MDString::get(Ctx,
"llvm.loop.vectorize.width"),
258 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
260 Args.push_back(MDNode::get(Ctx, Vals));
266 MDString::get(Ctx,
"llvm.loop.interleave.count"),
267 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
269 Args.push_back(MDNode::get(Ctx, Vals));
275 MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
276 ConstantAsMetadata::get(ConstantInt::get(
277 llvm::Type::getInt1Ty(Ctx),
279 Args.push_back(MDNode::get(Ctx, Vals));
282 if (FollowupHasTransforms)
283 Args.push_back(MDNode::get(
285 {MDString::get(Ctx,
"llvm.loop.vectorize.followup_all"), Followup}));
287 MDNode *LoopID = MDNode::get(Ctx, Args);
288 LoopID->replaceOperandWith(0, LoopID);
289 HasUserTransforms =
true;
294 LoopInfo::createLoopDistributeMetadata(
const LoopAttributes &Attrs,
296 bool &HasUserTransforms) {
297 LLVMContext &Ctx = Header->getContext();
305 if (Enabled !=
true) {
307 if (Enabled ==
false) {
308 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
309 NewLoopProperties.push_back(
310 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
311 ConstantAsMetadata::get(ConstantInt::get(
312 llvm::Type::getInt1Ty(Ctx), 0))}));
313 LoopProperties = NewLoopProperties;
315 return createLoopVectorizeMetadata(Attrs, LoopProperties,
319 bool FollowupHasTransforms =
false;
321 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
324 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
325 Args.push_back(TempNode.get());
326 Args.append(LoopProperties.begin(), LoopProperties.end());
328 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
329 ConstantAsMetadata::get(ConstantInt::get(
330 llvm::Type::getInt1Ty(Ctx),
332 Args.push_back(MDNode::get(Ctx, Vals));
334 if (FollowupHasTransforms)
335 Args.push_back(MDNode::get(
337 {MDString::get(Ctx,
"llvm.loop.distribute.followup_all"), Followup}));
339 MDNode *LoopID = MDNode::get(Ctx, Args);
340 LoopID->replaceOperandWith(0, LoopID);
341 HasUserTransforms =
true;
345 MDNode *LoopInfo::createFullUnrollMetadata(
const LoopAttributes &Attrs,
347 bool &HasUserTransforms) {
348 LLVMContext &Ctx = Header->getContext();
356 if (Enabled !=
true) {
358 if (Enabled ==
false) {
359 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
360 NewLoopProperties.push_back(
361 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
362 LoopProperties = NewLoopProperties;
364 return createLoopDistributeMetadata(Attrs, LoopProperties,
369 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
370 Args.push_back(TempNode.get());
371 Args.append(LoopProperties.begin(), LoopProperties.end());
372 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.full")));
377 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
378 LoopID->replaceOperandWith(0, LoopID);
379 HasUserTransforms =
true;
383 MDNode *LoopInfo::createMetadata(
386 bool &HasUserTransforms) {
391 LoopProperties.push_back(StartLoc.getAsMDNode());
395 LoopProperties.push_back(EndLoc.getAsMDNode());
399 "There must be an access group iff the loop is parallel");
401 LLVMContext &Ctx = Header->getContext();
402 LoopProperties.push_back(MDNode::get(
403 Ctx, {MDString::get(Ctx,
"llvm.loop.parallel_accesses"), AccGroup}));
406 LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
407 AdditionalLoopProperties.end());
408 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
415 InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
417 PipelineInitiationInterval(0) {}
434 const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc,
436 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
441 LLVMContext &Ctx = Header->getContext();
442 AccGroup = MDNode::getDistinct(Ctx, {});
456 TempLoopID = MDNode::getTemporary(Header->getContext(), None);
467 LLVMContext &Ctx = Header->getContext();
512 if (!Parent->UnrollAndJamInnerFollowup) {
520 BeforeLoopProperties.push_back(
521 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
523 bool InnerFollowupHasTransform =
false;
524 MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
525 InnerFollowupHasTransform);
526 if (InnerFollowupHasTransform)
527 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
530 CurLoopAttr = BeforeJam;
533 bool HasUserTransforms =
false;
534 LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
535 TempLoopID->replaceAllUsesWith(LoopID);
539 const llvm::DebugLoc &EndLoc) {
540 Active.push_back(
LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
541 Active.empty() ? nullptr : &Active.back()));
548 const llvm::DebugLoc &StartLoc,
549 const llvm::DebugLoc &EndLoc) {
552 for (
const auto *
Attr : Attrs) {
553 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(
Attr);
554 const OpenCLUnrollHintAttr *OpenCLHint =
555 dyn_cast<OpenCLUnrollHintAttr>(
Attr);
558 if (!LH && !OpenCLHint) {
562 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
563 LoopHintAttr::LoopHintState
State = LoopHintAttr::Disable;
564 unsigned ValueInt = 1;
572 ValueInt = OpenCLHint->getUnrollHint();
574 State = LoopHintAttr::Enable;
575 }
else if (ValueInt != 1) {
576 Option = LoopHintAttr::UnrollCount;
577 State = LoopHintAttr::Numeric;
580 auto *ValueExpr = LH->getValue();
582 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
583 ValueInt = ValueAPS.getSExtValue();
586 Option = LH->getOption();
587 State = LH->getState();
590 case LoopHintAttr::Disable:
592 case LoopHintAttr::Vectorize:
594 setVectorizeWidth(1);
596 case LoopHintAttr::Interleave:
598 setInterleaveCount(1);
600 case LoopHintAttr::Unroll:
603 case LoopHintAttr::UnrollAndJam:
606 case LoopHintAttr::Distribute:
607 setDistributeState(
false);
609 case LoopHintAttr::PipelineDisabled:
610 setPipelineDisabled(
true);
612 case LoopHintAttr::UnrollCount:
613 case LoopHintAttr::UnrollAndJamCount:
614 case LoopHintAttr::VectorizeWidth:
615 case LoopHintAttr::InterleaveCount:
616 case LoopHintAttr::PipelineInitiationInterval:
617 llvm_unreachable(
"Options cannot be disabled.");
621 case LoopHintAttr::Enable:
623 case LoopHintAttr::Vectorize:
624 case LoopHintAttr::Interleave:
625 setVectorizeEnable(
true);
627 case LoopHintAttr::Unroll:
630 case LoopHintAttr::UnrollAndJam:
633 case LoopHintAttr::Distribute:
634 setDistributeState(
true);
636 case LoopHintAttr::UnrollCount:
637 case LoopHintAttr::UnrollAndJamCount:
638 case LoopHintAttr::VectorizeWidth:
639 case LoopHintAttr::InterleaveCount:
640 case LoopHintAttr::PipelineDisabled:
641 case LoopHintAttr::PipelineInitiationInterval:
642 llvm_unreachable(
"Options cannot enabled.");
646 case LoopHintAttr::AssumeSafety:
648 case LoopHintAttr::Vectorize:
649 case LoopHintAttr::Interleave:
652 setVectorizeEnable(
true);
654 case LoopHintAttr::Unroll:
655 case LoopHintAttr::UnrollAndJam:
656 case LoopHintAttr::UnrollCount:
657 case LoopHintAttr::UnrollAndJamCount:
658 case LoopHintAttr::VectorizeWidth:
659 case LoopHintAttr::InterleaveCount:
660 case LoopHintAttr::Distribute:
661 case LoopHintAttr::PipelineDisabled:
662 case LoopHintAttr::PipelineInitiationInterval:
663 llvm_unreachable(
"Options cannot be used to assume mem safety.");
667 case LoopHintAttr::Full:
669 case LoopHintAttr::Unroll:
672 case LoopHintAttr::UnrollAndJam:
675 case LoopHintAttr::Vectorize:
676 case LoopHintAttr::Interleave:
677 case LoopHintAttr::UnrollCount:
678 case LoopHintAttr::UnrollAndJamCount:
679 case LoopHintAttr::VectorizeWidth:
680 case LoopHintAttr::InterleaveCount:
681 case LoopHintAttr::Distribute:
682 case LoopHintAttr::PipelineDisabled:
683 case LoopHintAttr::PipelineInitiationInterval:
684 llvm_unreachable(
"Options cannot be used with 'full' hint.");
688 case LoopHintAttr::Numeric:
690 case LoopHintAttr::VectorizeWidth:
691 setVectorizeWidth(ValueInt);
693 case LoopHintAttr::InterleaveCount:
694 setInterleaveCount(ValueInt);
696 case LoopHintAttr::UnrollCount:
697 setUnrollCount(ValueInt);
699 case LoopHintAttr::UnrollAndJamCount:
700 setUnrollAndJamCount(ValueInt);
702 case LoopHintAttr::PipelineInitiationInterval:
703 setPipelineInitiationInterval(ValueInt);
705 case LoopHintAttr::Unroll:
706 case LoopHintAttr::UnrollAndJam:
707 case LoopHintAttr::Vectorize:
708 case LoopHintAttr::Interleave:
709 case LoopHintAttr::Distribute:
710 case LoopHintAttr::PipelineDisabled:
711 llvm_unreachable(
"Options cannot be assigned a value.");
719 push(Header, StartLoc, EndLoc);
723 assert(!Active.empty() &&
"No active loops to pop");
724 Active.back().finish();
729 if (I->mayReadOrWriteMemory()) {
733 if (MDNode *Group = AL.getAccessGroup())
734 AccessGroups.push_back(Group);
736 MDNode *UnionMD =
nullptr;
737 if (AccessGroups.size() == 1)
738 UnionMD = cast<MDNode>(AccessGroups[0]);
739 else if (AccessGroups.size() >= 2)
740 UnionMD = MDNode::get(I->getContext(), AccessGroups);
741 I->setMetadata(
"llvm.access.group", UnionMD);
751 if (I->isTerminator()) {
752 for (BasicBlock *Succ : successors(I))
754 I->setMetadata(llvm::LLVMContext::MD_loop, L.
getLoopID());
Defines the clang::ASTContext interface.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Attributes that may be specified on loops.
unsigned UnrollAndJamCount
llvm.unroll.
Information used when generating a structured loop.
LoopAttributes(bool IsParallel=false)
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.
void pop()
End the current loop.
Whether values of this type can be null is (explicitly) unspecified.
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
void finish()
Create the loop's metadata.
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
unsigned UnrollCount
llvm.unroll.
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
constexpr XRayInstrMask None
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
static const TypeInfo & getInfo(unsigned id)
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
Attr - This represents one attribute.