23#define DEBUG_TYPE "llvm-mca"
30 NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));
36 uint64_t CandidateMask = ReadyMask & NextInSequenceMask;
38 return selectImpl(CandidateMask, NextInSequenceMask);
40 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;
41 RemovedFromNextInSequence = 0;
42 CandidateMask = ReadyMask & NextInSequenceMask;
44 return selectImpl(CandidateMask, NextInSequenceMask);
46 NextInSequenceMask = ResourceUnitMask;
47 CandidateMask = ReadyMask & NextInSequenceMask;
48 return selectImpl(CandidateMask, NextInSequenceMask);
52 if (Mask > NextInSequenceMask) {
53 RemovedFromNextInSequence |= Mask;
57 NextInSequenceMask &= (~Mask);
58 if (NextInSequenceMask)
61 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;
62 RemovedFromNextInSequence = 0;
67 : ProcResourceDescIndex(
Index), ResourceMask(Mask),
73 ResourceSizeMask = (1ULL <<
Desc.NumUnits) - 1;
75 ReadyMask = ResourceSizeMask;
76 AvailableSlots = BufferSize == -1 ? 0U :
static_cast<unsigned>(BufferSize);
96 <<
", SZMASK=" <<
format_hex(ResourceSizeMask, 16)
98 <<
", BufferSize=" << BufferSize
99 <<
", AvailableSlots=" << AvailableSlots
100 <<
", Reserved=" << Unavailable <<
'\n';
104static std::unique_ptr<ResourceStrategy>
107 return std::make_unique<DefaultResourceStrategy>(RS.
getReadyMask());
108 return std::unique_ptr<ResourceStrategy>(
nullptr);
112 : Resources(SM.getNumProcResourceKinds() - 1),
113 Strategies(SM.getNumProcResourceKinds() - 1),
114 Resource2Groups(SM.getNumProcResourceKinds() - 1, 0),
115 ProcResID2Mask(SM.getNumProcResourceKinds(), 0),
116 ResIndex2ProcResID(SM.getNumProcResourceKinds() - 1, 0),
117 ProcResUnitMask(0), ReservedResourceGroups(0), AvailableBuffers(~0ULL),
124 ResIndex2ProcResID[
Index] =
I;
140 ProcResUnitMask |= Mask;
145 Mask -= GroupMaskIdx;
150 Resource2Groups[IndexUnit] |= GroupMaskIdx;
155 AvailableProcResUnits = ProcResUnitMask;
158void ResourceManager::setCustomStrategyImpl(std::unique_ptr<ResourceStrategy> S,
161 assert(
Index < Resources.size() &&
"Invalid processor resource index!");
162 assert(S &&
"Unexpected null strategy in input!");
163 Strategies[
Index] = std::move(S);
170unsigned ResourceManager::getNumUnits(
uint64_t ResourceID)
const {
179 assert(
Index < Resources.size() &&
"Invalid resource use!");
180 ResourceState &RS = *Resources[
Index];
181 assert(RS.isReady() &&
"No available units to select!");
185 if (!RS.isAResourceGroup() && RS.getNumUnits() == 1)
186 return std::make_pair(ResourceID, RS.getReadyMask());
188 uint64_t SubResourceID = Strategies[
Index]->select(RS.getReadyMask());
189 if (RS.isAResourceGroup())
190 return selectPipe(SubResourceID);
191 return std::make_pair(ResourceID, SubResourceID);
197 ResourceState &RS = *Resources[RSID];
198 RS.markSubResourceAsUsed(RR.second);
201 if (RS.getNumUnits() > 1)
202 Strategies[RSID]->used(RR.second);
209 AvailableProcResUnits ^= RR.first;
216 ResourceState &CurrentUser = *Resources[GroupIndex];
217 CurrentUser.markSubResourceAsUsed(RR.first);
218 Strategies[GroupIndex]->used(RR.first);
224void ResourceManager::release(
const ResourceRef &RR) {
226 ResourceState &RS = *Resources[RSID];
227 bool WasFullyUsed = !RS.isReady();
228 RS.releaseSubResource(RR.second);
232 AvailableProcResUnits ^= RR.first;
238 ResourceState &CurrentUser = *Resources[GroupIndex];
239 CurrentUser.releaseSubResource(RR.first);
246 if (ConsumedBuffers & ReservedBuffers)
248 if (ConsumedBuffers & (~AvailableBuffers))
254 while (ConsumedBuffers) {
255 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
257 ConsumedBuffers ^= CurrentBuffer;
259 if (!RS.reserveBuffer())
260 AvailableBuffers ^= CurrentBuffer;
261 if (RS.isADispatchHazard()) {
265 ReservedBuffers ^= CurrentBuffer;
271 AvailableBuffers |= ConsumedBuffers;
272 while (ConsumedBuffers) {
273 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
275 ConsumedBuffers ^= CurrentBuffer;
287 for (
const std::pair<uint64_t, ResourceUsage> &
E :
Desc.Resources) {
288 unsigned NumUnits =
E.second.isReserved() ? 0U :
E.second.NumUnits;
291 BusyResourceMask |=
E.first;
297 NumAvailableUnits -= NumUnits;
298 AvailableUnits[
E.first] = NumAvailableUnits;
299 if (!NumAvailableUnits)
300 ConsumedResourceMask |=
E.first;
304 BusyResourceMask &= ProcResUnitMask;
305 if (BusyResourceMask)
306 return BusyResourceMask;
308 BusyResourceMask =
Desc.UsedProcResGroups & ReservedResourceGroups;
309 if (!
Desc.HasPartiallyOverlappingGroups || BusyResourceMask)
310 return BusyResourceMask;
314 for (
const std::pair<uint64_t, ResourceUsage> &
E :
Desc.Resources) {
325 auto it = AvailableUnits.
find(ResourceMask);
326 if (it == AvailableUnits.
end()) {
330 AvailableUnits.
insert(std::make_pair(ResourceMask, NumUnits)).first;
334 BusyResourceMask |= it->first;
340 ConsumedResourceMask |= it->first;
344 return BusyResourceMask;
350 for (
const std::pair<uint64_t, ResourceUsage> &R :
Desc.Resources) {
357 assert(CS.
begin() == 0 &&
"Invalid {Start, End} cycles!");
358 if (!R.second.isReserved()) {
361 BusyResources[Pipe] += CS.
size();
362 Pipes.emplace_back(std::pair<ResourceRef, ReleaseAtCycles>(
367 assert(R.second.isReserved());
375 for (std::pair<ResourceRef, unsigned> &BR : BusyResources) {
390 BusyResources.erase(RF);
397 "Unexpected resource state found!");
399 ReservedResourceGroups ^= 1ULL <<
Index;
407 ReservedResourceGroups ^= 1ULL <<
Index;
410 ReservedBuffers ^= 1ULL <<
Index;
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
iv Induction Variable Users
Move duplicate certain instructions close to their use
The classes here represent processor resource units and their management strategy.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
void used(uint64_t Mask) override
Called by the ResourceManager when a processor resource group, or a processor resource with multiple ...
uint64_t select(uint64_t ReadyMask) override
Selects a processor resource unit from a ReadyMask.
This class represents the number of cycles per resource (fractions of cycles).
void reserveBuffers(uint64_t ConsumedBuffers)
void releaseBuffers(uint64_t ConsumedBuffers)
void issueInstruction(const InstrDesc &Desc, SmallVectorImpl< std::pair< ResourceRef, ReleaseAtCycles > > &Pipes)
void releaseResource(uint64_t ResourceID)
ResourceManager(const MCSchedModel &SM)
void reserveResource(uint64_t ResourceID)
unsigned resolveResourceMask(uint64_t Mask) const
void cycleEvent(SmallVectorImpl< ResourceRef > &ResourcesFreed)
ResourceStateEvent canBeDispatched(uint64_t ConsumedBuffers) const
uint64_t checkAvailability(const InstrDesc &Desc) const
A processor resource descriptor.
ResourceStateEvent isBufferAvailable() const
Checks if there is an available slot in the resource buffer.
ResourceState(const MCProcResourceDesc &Desc, unsigned Index, uint64_t Mask)
uint64_t getReadyMask() const
bool isADispatchHazard() const
Returns true if this is an in-order dispatch/issue resource.
bool isReady(unsigned NumUnits=1) const
Returs true if this resource is not reserved, and if there are at least NumUnits available units.
unsigned getNumUnits() const
bool isAResourceGroup() const
virtual ~ResourceStrategy()
Helper functions used by various pipeline components.
static std::unique_ptr< ResourceStrategy > getStrategyFor(const ResourceState &RS)
ResourceStateEvent
Used to notify the internal state of a processor resource.
static uint64_t selectImpl(uint64_t CandidateMask, uint64_t &NextInSequenceMask)
std::pair< uint64_t, uint64_t > ResourceRef
A resource unit identifier.
void computeProcResourceMasks(const MCSchedModel &SM, MutableArrayRef< uint64_t > Masks)
Populates vector Masks with processor resource masks.
unsigned getResourceStateIndex(uint64_t Mask)
This is an optimization pass for GlobalISel generic memory operations.
int popcount(T Value) noexcept
Count the number of set bits in a value.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FormattedNumber format_hex(uint64_t N, unsigned Width, bool Upper=false)
format_hex - Output N as a fixed width hexadecimal.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Description of the encoding of one expression Op.
Define a kind of processor resource that will be modeled by the scheduler.
Machine model for scheduling, bundling, and heuristics.
unsigned getNumProcResourceKinds() const
const MCProcResourceDesc * getProcResource(unsigned ProcResourceIdx) const
An instruction descriptor.