23 #define DEBUG_TYPE "llvm-mca"
27 uint64_t &NextInSequenceMask) {
30 NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));
36 uint64_t CandidateMask = ReadyMask & NextInSequenceMask;
38 return selectImpl(CandidateMask, NextInSequenceMask);
40 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;
41 RemovedFromNextInSequence = 0;
42 CandidateMask = ReadyMask & NextInSequenceMask;
44 return selectImpl(CandidateMask, NextInSequenceMask);
46 NextInSequenceMask = ResourceUnitMask;
47 CandidateMask = ReadyMask & NextInSequenceMask;
48 return selectImpl(CandidateMask, NextInSequenceMask);
52 if (
Mask > NextInSequenceMask) {
53 RemovedFromNextInSequence |=
Mask;
57 NextInSequenceMask &= (~
Mask);
58 if (NextInSequenceMask)
61 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;
62 RemovedFromNextInSequence = 0;
67 : ProcResourceDescIndex(
Index), ResourceMask(
Mask),
68 BufferSize(Desc.BufferSize), IsAGroup(
countPopulation(ResourceMask) > 1) {
73 ResourceSizeMask = (1ULL << Desc.
NumUnits) - 1;
75 ReadyMask = ResourceSizeMask;
76 AvailableSlots = BufferSize == -1 ? 0U :
static_cast<unsigned>(BufferSize);
96 <<
", SZMASK=" <<
format_hex(ResourceSizeMask, 16)
98 <<
", BufferSize=" << BufferSize
99 <<
", AvailableSlots=" << AvailableSlots
100 <<
", Reserved=" << Unavailable <<
'\n';
104 static std::unique_ptr<ResourceStrategy>
107 return std::make_unique<DefaultResourceStrategy>(RS.
getReadyMask());
108 return std::unique_ptr<ResourceStrategy>(
nullptr);
112 : Resources(SM.getNumProcResourceKinds() - 1),
113 Strategies(SM.getNumProcResourceKinds() - 1),
114 Resource2Groups(SM.getNumProcResourceKinds() - 1, 0),
115 ProcResID2Mask(SM.getNumProcResourceKinds(), 0),
116 ResIndex2ProcResID(SM.getNumProcResourceKinds() - 1, 0),
117 ProcResUnitMask(0), ReservedResourceGroups(0),
118 AvailableBuffers(~0ULL), ReservedBuffers(0) {
124 ResIndex2ProcResID[
Index] =
I;
128 uint64_t
Mask = ProcResID2Mask[
I];
136 uint64_t
Mask = ProcResID2Mask[
I];
140 ProcResUnitMask |=
Mask;
144 uint64_t GroupMaskIdx = 1ULL <<
Index;
145 Mask -= GroupMaskIdx;
150 Resource2Groups[IndexUnit] |= GroupMaskIdx;
155 AvailableProcResUnits = ProcResUnitMask;
158 void ResourceManager::setCustomStrategyImpl(std::unique_ptr<ResourceStrategy>
S,
159 uint64_t ResourceMask) {
161 assert(
Index < Resources.size() &&
"Invalid processor resource index!");
162 assert(
S &&
"Unexpected null strategy in input!");
170 unsigned ResourceManager::getNumUnits(uint64_t ResourceID)
const {
177 ResourceRef ResourceManager::selectPipe(uint64_t ResourceID) {
179 assert(
Index < Resources.size() &&
"Invalid resource use!");
180 ResourceState &RS = *Resources[
Index];
181 assert(RS.isReady() &&
"No available units to select!");
185 if (!RS.isAResourceGroup() && RS.getNumUnits() == 1)
186 return std::make_pair(ResourceID, RS.getReadyMask());
188 uint64_t SubResourceID = Strategies[
Index]->select(RS.getReadyMask());
189 if (RS.isAResourceGroup())
190 return selectPipe(SubResourceID);
191 return std::make_pair(ResourceID, SubResourceID);
197 ResourceState &RS = *Resources[RSID];
198 RS.markSubResourceAsUsed(RR.second);
201 if (RS.getNumUnits() > 1)
202 Strategies[RSID]->used(RR.second);
209 AvailableProcResUnits ^= RR.first;
212 uint64_t
Users = Resource2Groups[RSID];
216 ResourceState &CurrentUser = *Resources[GroupIndex];
217 CurrentUser.markSubResourceAsUsed(RR.first);
218 Strategies[GroupIndex]->used(RR.first);
224 void ResourceManager::release(
const ResourceRef &RR) {
226 ResourceState &RS = *Resources[RSID];
227 bool WasFullyUsed = !RS.isReady();
228 RS.releaseSubResource(RR.second);
232 AvailableProcResUnits ^= RR.first;
235 uint64_t
Users = Resource2Groups[RSID];
238 ResourceState &CurrentUser = *Resources[GroupIndex];
239 CurrentUser.releaseSubResource(RR.first);
246 if (ConsumedBuffers & ReservedBuffers)
248 if (ConsumedBuffers & (~AvailableBuffers))
254 while (ConsumedBuffers) {
255 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
257 ConsumedBuffers ^= CurrentBuffer;
259 if (!RS.reserveBuffer())
260 AvailableBuffers ^= CurrentBuffer;
261 if (RS.isADispatchHazard()) {
265 ReservedBuffers ^= CurrentBuffer;
271 AvailableBuffers |= ConsumedBuffers;
272 while (ConsumedBuffers) {
273 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
275 ConsumedBuffers ^= CurrentBuffer;
283 uint64_t BusyResourceMask = 0;
284 for (
const std::pair<uint64_t, ResourceUsage> &
E : Desc.
Resources) {
285 unsigned NumUnits =
E.second.isReserved() ? 0U :
E.second.NumUnits;
287 if (!Resources[
Index]->isReady(NumUnits))
288 BusyResourceMask |=
E.first;
291 BusyResourceMask &= ProcResUnitMask;
292 if (BusyResourceMask)
293 return BusyResourceMask;
300 for (
const std::pair<uint64_t, ResourceUsage> &R : Desc.
Resources) {
307 assert(CS.
begin() == 0 &&
"Invalid {Start, End} cycles!");
308 if (!R.second.isReserved()) {
311 BusyResources[Pipe] += CS.
size();
312 Pipes.emplace_back(std::pair<ResourceRef, ResourceCycles>(
317 assert(R.second.isReserved());
325 for (std::pair<ResourceRef, unsigned> &
BR : BusyResources) {
335 ResourcesFreed.push_back(RR);
340 BusyResources.
erase(RF);
347 "Unexpected resource state found!");
349 ReservedResourceGroups ^= 1ULL <<
Index;
357 ReservedResourceGroups ^= 1ULL <<
Index;
360 ReservedBuffers ^= 1ULL <<
Index;