LLVM 17.0.0git
TargetInstrInfo.cpp
Go to the documentation of this file.
1//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
30#include "llvm/IR/DataLayout.h"
32#include "llvm/MC/MCAsmInfo.h"
37
38using namespace llvm;
39
41 "disable-sched-hazard", cl::Hidden, cl::init(false),
42 cl::desc("Disable hazard detection during preRA scheduling"));
43
45
47TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum,
49 const MachineFunction &MF) const {
50 if (OpNum >= MCID.getNumOperands())
51 return nullptr;
52
53 short RegClass = MCID.operands()[OpNum].RegClass;
54 if (MCID.operands()[OpNum].isLookupPtrRegClass())
55 return TRI->getPointerRegClass(MF, RegClass);
56
57 // Instructions like INSERT_SUBREG do not have fixed register classes.
58 if (RegClass < 0)
59 return nullptr;
60
61 // Otherwise just look it up normally.
62 return TRI->getRegClass(RegClass);
63}
64
65/// insertNoop - Insert a noop into the instruction stream at the specified
66/// point.
69 llvm_unreachable("Target didn't implement insertNoop!");
70}
71
72/// insertNoops - Insert noops into the instruction stream at the specified
73/// point.
76 unsigned Quantity) const {
77 for (unsigned i = 0; i < Quantity; ++i)
79}
80
81static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) {
82 return strncmp(Str, MAI.getCommentString().data(),
83 MAI.getCommentString().size()) == 0;
84}
85
86/// Measure the specified inline asm to determine an approximation of its
87/// length.
88/// Comments (which run till the next SeparatorString or newline) do not
89/// count as an instruction.
90/// Any other non-whitespace text is considered an instruction, with
91/// multiple instructions separated by SeparatorString or newlines.
92/// Variable-length instructions are not handled here; this function
93/// may be overloaded in the target code to do that.
94/// We implement a special case of the .space directive which takes only a
95/// single integer argument in base 10 that is the size in bytes. This is a
96/// restricted form of the GAS directive in that we only interpret
97/// simple--i.e. not a logical or arithmetic expression--size values without
98/// the optional fill value. This is primarily used for creating arbitrary
99/// sized inline asm blocks for testing purposes.
101 const char *Str,
102 const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const {
103 // Count the number of instructions in the asm.
104 bool AtInsnStart = true;
105 unsigned Length = 0;
106 const unsigned MaxInstLength = MAI.getMaxInstLength(STI);
107 for (; *Str; ++Str) {
108 if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(),
109 strlen(MAI.getSeparatorString())) == 0) {
110 AtInsnStart = true;
111 } else if (isAsmComment(Str, MAI)) {
112 // Stop counting as an instruction after a comment until the next
113 // separator.
114 AtInsnStart = false;
115 }
116
117 if (AtInsnStart && !isSpace(static_cast<unsigned char>(*Str))) {
118 unsigned AddLength = MaxInstLength;
119 if (strncmp(Str, ".space", 6) == 0) {
120 char *EStr;
121 int SpaceSize;
122 SpaceSize = strtol(Str + 6, &EStr, 10);
123 SpaceSize = SpaceSize < 0 ? 0 : SpaceSize;
124 while (*EStr != '\n' && isSpace(static_cast<unsigned char>(*EStr)))
125 ++EStr;
126 if (*EStr == '\0' || *EStr == '\n' ||
127 isAsmComment(EStr, MAI)) // Successfully parsed .space argument
128 AddLength = SpaceSize;
129 }
130 Length += AddLength;
131 AtInsnStart = false;
132 }
133 }
134
135 return Length;
136}
137
138/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
139/// after it, replacing it with an unconditional branch to NewDest.
140void
142 MachineBasicBlock *NewDest) const {
143 MachineBasicBlock *MBB = Tail->getParent();
144
145 // Remove all the old successors of MBB from the CFG.
146 while (!MBB->succ_empty())
148
149 // Save off the debug loc before erasing the instruction.
150 DebugLoc DL = Tail->getDebugLoc();
151
152 // Update call site info and remove all the dead instructions
153 // from the end of MBB.
154 while (Tail != MBB->end()) {
155 auto MI = Tail++;
156 if (MI->shouldUpdateCallSiteInfo())
158 MBB->erase(MI);
159 }
160
161 // If MBB isn't immediately before MBB, insert a branch to it.
163 insertBranch(*MBB, NewDest, nullptr, SmallVector<MachineOperand, 0>(), DL);
164 MBB->addSuccessor(NewDest);
165}
166
168 bool NewMI, unsigned Idx1,
169 unsigned Idx2) const {
170 const MCInstrDesc &MCID = MI.getDesc();
171 bool HasDef = MCID.getNumDefs();
172 if (HasDef && !MI.getOperand(0).isReg())
173 // No idea how to commute this instruction. Target should implement its own.
174 return nullptr;
175
176 unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1;
177 unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2;
178 assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) &&
179 CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 &&
180 "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands.");
181 assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() &&
182 "This only knows how to commute register operands so far");
183
184 Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register();
185 Register Reg1 = MI.getOperand(Idx1).getReg();
186 Register Reg2 = MI.getOperand(Idx2).getReg();
187 unsigned SubReg0 = HasDef ? MI.getOperand(0).getSubReg() : 0;
188 unsigned SubReg1 = MI.getOperand(Idx1).getSubReg();
189 unsigned SubReg2 = MI.getOperand(Idx2).getSubReg();
190 bool Reg1IsKill = MI.getOperand(Idx1).isKill();
191 bool Reg2IsKill = MI.getOperand(Idx2).isKill();
192 bool Reg1IsUndef = MI.getOperand(Idx1).isUndef();
193 bool Reg2IsUndef = MI.getOperand(Idx2).isUndef();
194 bool Reg1IsInternal = MI.getOperand(Idx1).isInternalRead();
195 bool Reg2IsInternal = MI.getOperand(Idx2).isInternalRead();
196 // Avoid calling isRenamable for virtual registers since we assert that
197 // renamable property is only queried/set for physical registers.
198 bool Reg1IsRenamable =
199 Reg1.isPhysical() ? MI.getOperand(Idx1).isRenamable() : false;
200 bool Reg2IsRenamable =
201 Reg2.isPhysical() ? MI.getOperand(Idx2).isRenamable() : false;
202 // If destination is tied to either of the commuted source register, then
203 // it must be updated.
204 if (HasDef && Reg0 == Reg1 &&
205 MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) {
206 Reg2IsKill = false;
207 Reg0 = Reg2;
208 SubReg0 = SubReg2;
209 } else if (HasDef && Reg0 == Reg2 &&
210 MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) {
211 Reg1IsKill = false;
212 Reg0 = Reg1;
213 SubReg0 = SubReg1;
214 }
215
216 MachineInstr *CommutedMI = nullptr;
217 if (NewMI) {
218 // Create a new instruction.
219 MachineFunction &MF = *MI.getMF();
220 CommutedMI = MF.CloneMachineInstr(&MI);
221 } else {
222 CommutedMI = &MI;
223 }
224
225 if (HasDef) {
226 CommutedMI->getOperand(0).setReg(Reg0);
227 CommutedMI->getOperand(0).setSubReg(SubReg0);
228 }
229 CommutedMI->getOperand(Idx2).setReg(Reg1);
230 CommutedMI->getOperand(Idx1).setReg(Reg2);
231 CommutedMI->getOperand(Idx2).setSubReg(SubReg1);
232 CommutedMI->getOperand(Idx1).setSubReg(SubReg2);
233 CommutedMI->getOperand(Idx2).setIsKill(Reg1IsKill);
234 CommutedMI->getOperand(Idx1).setIsKill(Reg2IsKill);
235 CommutedMI->getOperand(Idx2).setIsUndef(Reg1IsUndef);
236 CommutedMI->getOperand(Idx1).setIsUndef(Reg2IsUndef);
237 CommutedMI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal);
238 CommutedMI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal);
239 // Avoid calling setIsRenamable for virtual registers since we assert that
240 // renamable property is only queried/set for physical registers.
241 if (Reg1.isPhysical())
242 CommutedMI->getOperand(Idx2).setIsRenamable(Reg1IsRenamable);
243 if (Reg2.isPhysical())
244 CommutedMI->getOperand(Idx1).setIsRenamable(Reg2IsRenamable);
245 return CommutedMI;
246}
247
249 unsigned OpIdx1,
250 unsigned OpIdx2) const {
251 // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose
252 // any commutable operand, which is done in findCommutedOpIndices() method
253 // called below.
254 if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) &&
255 !findCommutedOpIndices(MI, OpIdx1, OpIdx2)) {
256 assert(MI.isCommutable() &&
257 "Precondition violation: MI must be commutable.");
258 return nullptr;
259 }
260 return commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
261}
262
264 unsigned &ResultIdx2,
265 unsigned CommutableOpIdx1,
266 unsigned CommutableOpIdx2) {
267 if (ResultIdx1 == CommuteAnyOperandIndex &&
268 ResultIdx2 == CommuteAnyOperandIndex) {
269 ResultIdx1 = CommutableOpIdx1;
270 ResultIdx2 = CommutableOpIdx2;
271 } else if (ResultIdx1 == CommuteAnyOperandIndex) {
272 if (ResultIdx2 == CommutableOpIdx1)
273 ResultIdx1 = CommutableOpIdx2;
274 else if (ResultIdx2 == CommutableOpIdx2)
275 ResultIdx1 = CommutableOpIdx1;
276 else
277 return false;
278 } else if (ResultIdx2 == CommuteAnyOperandIndex) {
279 if (ResultIdx1 == CommutableOpIdx1)
280 ResultIdx2 = CommutableOpIdx2;
281 else if (ResultIdx1 == CommutableOpIdx2)
282 ResultIdx2 = CommutableOpIdx1;
283 else
284 return false;
285 } else
286 // Check that the result operand indices match the given commutable
287 // operand indices.
288 return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) ||
289 (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1);
290
291 return true;
292}
293
295 unsigned &SrcOpIdx1,
296 unsigned &SrcOpIdx2) const {
297 assert(!MI.isBundle() &&
298 "TargetInstrInfo::findCommutedOpIndices() can't handle bundles");
299
300 const MCInstrDesc &MCID = MI.getDesc();
301 if (!MCID.isCommutable())
302 return false;
303
304 // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this
305 // is not true, then the target must implement this.
306 unsigned CommutableOpIdx1 = MCID.getNumDefs();
307 unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1;
308 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
309 CommutableOpIdx1, CommutableOpIdx2))
310 return false;
311
312 if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg())
313 // No idea.
314 return false;
315 return true;
316}
317
319 if (!MI.isTerminator()) return false;
320
321 // Conditional branch is a special case.
322 if (MI.isBranch() && !MI.isBarrier())
323 return true;
324 if (!MI.isPredicable())
325 return true;
326 return !isPredicated(MI);
327}
328
331 bool MadeChange = false;
332
333 assert(!MI.isBundle() &&
334 "TargetInstrInfo::PredicateInstruction() can't handle bundles");
335
336 const MCInstrDesc &MCID = MI.getDesc();
337 if (!MI.isPredicable())
338 return false;
339
340 for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) {
341 if (MCID.operands()[i].isPredicate()) {
342 MachineOperand &MO = MI.getOperand(i);
343 if (MO.isReg()) {
344 MO.setReg(Pred[j].getReg());
345 MadeChange = true;
346 } else if (MO.isImm()) {
347 MO.setImm(Pred[j].getImm());
348 MadeChange = true;
349 } else if (MO.isMBB()) {
350 MO.setMBB(Pred[j].getMBB());
351 MadeChange = true;
352 }
353 ++j;
354 }
355 }
356 return MadeChange;
357}
358
360 const MachineInstr &MI,
362 size_t StartSize = Accesses.size();
363 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
364 oe = MI.memoperands_end();
365 o != oe; ++o) {
366 if ((*o)->isLoad() &&
367 isa_and_nonnull<FixedStackPseudoSourceValue>((*o)->getPseudoValue()))
368 Accesses.push_back(*o);
369 }
370 return Accesses.size() != StartSize;
371}
372
374 const MachineInstr &MI,
376 size_t StartSize = Accesses.size();
377 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
378 oe = MI.memoperands_end();
379 o != oe; ++o) {
380 if ((*o)->isStore() &&
381 isa_and_nonnull<FixedStackPseudoSourceValue>((*o)->getPseudoValue()))
382 Accesses.push_back(*o);
383 }
384 return Accesses.size() != StartSize;
385}
386
388 unsigned SubIdx, unsigned &Size,
389 unsigned &Offset,
390 const MachineFunction &MF) const {
392 if (!SubIdx) {
393 Size = TRI->getSpillSize(*RC);
394 Offset = 0;
395 return true;
396 }
397 unsigned BitSize = TRI->getSubRegIdxSize(SubIdx);
398 // Convert bit size to byte size.
399 if (BitSize % 8)
400 return false;
401
402 int BitOffset = TRI->getSubRegIdxOffset(SubIdx);
403 if (BitOffset < 0 || BitOffset % 8)
404 return false;
405
406 Size = BitSize / 8;
407 Offset = (unsigned)BitOffset / 8;
408
409 assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range");
410
411 if (!MF.getDataLayout().isLittleEndian()) {
412 Offset = TRI->getSpillSize(*RC) - (Offset + Size);
413 }
414 return true;
415}
416
419 Register DestReg, unsigned SubIdx,
420 const MachineInstr &Orig,
421 const TargetRegisterInfo &TRI) const {
423 MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI);
424 MBB.insert(I, MI);
425}
426
428 const MachineInstr &MI1,
429 const MachineRegisterInfo *MRI) const {
431}
432
434 MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const {
435 assert(!Orig.isNotDuplicable() && "Instruction cannot be duplicated");
437 return MF.cloneMachineInstrBundle(MBB, InsertBefore, Orig);
438}
439
440// If the COPY instruction in MI can be folded to a stack operation, return
441// the register class to use.
443 unsigned FoldIdx) {
444 assert(MI.isCopy() && "MI must be a COPY instruction");
445 if (MI.getNumOperands() != 2)
446 return nullptr;
447 assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand");
448
449 const MachineOperand &FoldOp = MI.getOperand(FoldIdx);
450 const MachineOperand &LiveOp = MI.getOperand(1 - FoldIdx);
451
452 if (FoldOp.getSubReg() || LiveOp.getSubReg())
453 return nullptr;
454
455 Register FoldReg = FoldOp.getReg();
456 Register LiveReg = LiveOp.getReg();
457
458 assert(FoldReg.isVirtual() && "Cannot fold physregs");
459
460 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
461 const TargetRegisterClass *RC = MRI.getRegClass(FoldReg);
462
463 if (LiveOp.getReg().isPhysical())
464 return RC->contains(LiveOp.getReg()) ? RC : nullptr;
465
466 if (RC->hasSubClassEq(MRI.getRegClass(LiveReg)))
467 return RC;
468
469 // FIXME: Allow folding when register classes are memory compatible.
470 return nullptr;
471}
472
473MCInst TargetInstrInfo::getNop() const { llvm_unreachable("Not implemented"); }
474
475std::pair<unsigned, unsigned>
477 switch (MI.getOpcode()) {
478 case TargetOpcode::STACKMAP:
479 // StackMapLiveValues are foldable
480 return std::make_pair(0, StackMapOpers(&MI).getVarIdx());
481 case TargetOpcode::PATCHPOINT:
482 // For PatchPoint, the call args are not foldable (even if reported in the
483 // stackmap e.g. via anyregcc).
484 return std::make_pair(0, PatchPointOpers(&MI).getVarIdx());
485 case TargetOpcode::STATEPOINT:
486 // For statepoints, fold deopt and gc arguments, but not call arguments.
487 return std::make_pair(MI.getNumDefs(), StatepointOpers(&MI).getVarIdx());
488 default:
489 llvm_unreachable("unexpected stackmap opcode");
490 }
491}
492
494 ArrayRef<unsigned> Ops, int FrameIndex,
495 const TargetInstrInfo &TII) {
496 unsigned StartIdx = 0;
497 unsigned NumDefs = 0;
498 // getPatchpointUnfoldableRange throws guarantee if MI is not a patchpoint.
499 std::tie(NumDefs, StartIdx) = TII.getPatchpointUnfoldableRange(MI);
500
501 unsigned DefToFoldIdx = MI.getNumOperands();
502
503 // Return false if any operands requested for folding are not foldable (not
504 // part of the stackmap's live values).
505 for (unsigned Op : Ops) {
506 if (Op < NumDefs) {
507 assert(DefToFoldIdx == MI.getNumOperands() && "Folding multiple defs");
508 DefToFoldIdx = Op;
509 } else if (Op < StartIdx) {
510 return nullptr;
511 }
512 if (MI.getOperand(Op).isTied())
513 return nullptr;
514 }
515
516 MachineInstr *NewMI =
517 MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);
518 MachineInstrBuilder MIB(MF, NewMI);
519
520 // No need to fold return, the meta data, and function arguments
521 for (unsigned i = 0; i < StartIdx; ++i)
522 if (i != DefToFoldIdx)
523 MIB.add(MI.getOperand(i));
524
525 for (unsigned i = StartIdx, e = MI.getNumOperands(); i < e; ++i) {
526 MachineOperand &MO = MI.getOperand(i);
527 unsigned TiedTo = e;
528 (void)MI.isRegTiedToDefOperand(i, &TiedTo);
529
530 if (is_contained(Ops, i)) {
531 assert(TiedTo == e && "Cannot fold tied operands");
532 unsigned SpillSize;
533 unsigned SpillOffset;
534 // Compute the spill slot size and offset.
535 const TargetRegisterClass *RC =
536 MF.getRegInfo().getRegClass(MO.getReg());
537 bool Valid =
538 TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF);
539 if (!Valid)
540 report_fatal_error("cannot spill patchpoint subregister operand");
541 MIB.addImm(StackMaps::IndirectMemRefOp);
542 MIB.addImm(SpillSize);
543 MIB.addFrameIndex(FrameIndex);
544 MIB.addImm(SpillOffset);
545 } else {
546 MIB.add(MO);
547 if (TiedTo < e) {
548 assert(TiedTo < NumDefs && "Bad tied operand");
549 if (TiedTo > DefToFoldIdx)
550 --TiedTo;
551 NewMI->tieOperands(TiedTo, NewMI->getNumOperands() - 1);
552 }
553 }
554 }
555 return NewMI;
556}
557
559 ArrayRef<unsigned> Ops, int FI,
560 LiveIntervals *LIS,
561 VirtRegMap *VRM) const {
563 for (unsigned OpIdx : Ops)
564 Flags |= MI.getOperand(OpIdx).isDef() ? MachineMemOperand::MOStore
566
567 MachineBasicBlock *MBB = MI.getParent();
568 assert(MBB && "foldMemoryOperand needs an inserted instruction");
569 MachineFunction &MF = *MBB->getParent();
570
571 // If we're not folding a load into a subreg, the size of the load is the
572 // size of the spill slot. But if we are, we need to figure out what the
573 // actual load size is.
574 int64_t MemSize = 0;
575 const MachineFrameInfo &MFI = MF.getFrameInfo();
577
579 MemSize = MFI.getObjectSize(FI);
580 } else {
581 for (unsigned OpIdx : Ops) {
582 int64_t OpSize = MFI.getObjectSize(FI);
583
584 if (auto SubReg = MI.getOperand(OpIdx).getSubReg()) {
585 unsigned SubRegSize = TRI->getSubRegIdxSize(SubReg);
586 if (SubRegSize > 0 && !(SubRegSize % 8))
587 OpSize = SubRegSize / 8;
588 }
589
590 MemSize = std::max(MemSize, OpSize);
591 }
592 }
593
594 assert(MemSize && "Did not expect a zero-sized stack slot");
595
596 MachineInstr *NewMI = nullptr;
597
598 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
599 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
600 MI.getOpcode() == TargetOpcode::STATEPOINT) {
601 // Fold stackmap/patchpoint.
602 NewMI = foldPatchpoint(MF, MI, Ops, FI, *this);
603 if (NewMI)
604 MBB->insert(MI, NewMI);
605 } else {
606 // Ask the target to do the actual folding.
607 NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS, VRM);
608 }
609
610 if (NewMI) {
611 NewMI->setMemRefs(MF, MI.memoperands());
612 // Add a memory operand, foldMemoryOperandImpl doesn't do that.
614 NewMI->mayStore()) &&
615 "Folded a def to a non-store!");
617 NewMI->mayLoad()) &&
618 "Folded a use to a non-load!");
619 assert(MFI.getObjectOffset(FI) != -1);
620 MachineMemOperand *MMO =
622 Flags, MemSize, MFI.getObjectAlign(FI));
623 NewMI->addMemOperand(MF, MMO);
624
625 // The pass "x86 speculative load hardening" always attaches symbols to
626 // call instructions. We need copy it form old instruction.
627 NewMI->cloneInstrSymbols(MF, MI);
628
629 return NewMI;
630 }
631
632 // Straight COPY may fold as load/store.
633 if (!MI.isCopy() || Ops.size() != 1)
634 return nullptr;
635
636 const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]);
637 if (!RC)
638 return nullptr;
639
640 const MachineOperand &MO = MI.getOperand(1 - Ops[0]);
642
644 storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI,
645 Register());
646 else
647 loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI, Register());
648 return &*--Pos;
649}
650
653 MachineInstr &LoadMI,
654 LiveIntervals *LIS) const {
655 assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!");
656#ifndef NDEBUG
657 for (unsigned OpIdx : Ops)
658 assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!");
659#endif
660
661 MachineBasicBlock &MBB = *MI.getParent();
663
664 // Ask the target to do the actual folding.
665 MachineInstr *NewMI = nullptr;
666 int FrameIndex = 0;
667
668 if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
669 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
670 MI.getOpcode() == TargetOpcode::STATEPOINT) &&
671 isLoadFromStackSlot(LoadMI, FrameIndex)) {
672 // Fold stackmap/patchpoint.
673 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
674 if (NewMI)
675 NewMI = &*MBB.insert(MI, NewMI);
676 } else {
677 // Ask the target to do the actual folding.
678 NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, LIS);
679 }
680
681 if (!NewMI)
682 return nullptr;
683
684 // Copy the memoperands from the load to the folded instruction.
685 if (MI.memoperands_empty()) {
686 NewMI->setMemRefs(MF, LoadMI.memoperands());
687 } else {
688 // Handle the rare case of folding multiple loads.
689 NewMI->setMemRefs(MF, MI.memoperands());
691 E = LoadMI.memoperands_end();
692 I != E; ++I) {
693 NewMI->addMemOperand(MF, *I);
694 }
695 }
696 return NewMI;
697}
698
700 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
701 const MachineOperand &Op1 = Inst.getOperand(1);
702 const MachineOperand &Op2 = Inst.getOperand(2);
704
705 // We need virtual register definitions for the operands that we will
706 // reassociate.
707 MachineInstr *MI1 = nullptr;
708 MachineInstr *MI2 = nullptr;
709 if (Op1.isReg() && Op1.getReg().isVirtual())
710 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
711 if (Op2.isReg() && Op2.getReg().isVirtual())
712 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
713
714 // And at least one operand must be defined in MBB.
715 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
716}
717
719 unsigned Opcode2) const {
720 return Opcode1 == Opcode2 || getInverseOpcode(Opcode1) == Opcode2;
721}
722
724 bool &Commuted) const {
725 const MachineBasicBlock *MBB = Inst.getParent();
727 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
728 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
729 unsigned Opcode = Inst.getOpcode();
730
731 // If only one operand has the same or inverse opcode and it's the second
732 // source operand, the operands must be commuted.
733 Commuted = !areOpcodesEqualOrInverse(Opcode, MI1->getOpcode()) &&
734 areOpcodesEqualOrInverse(Opcode, MI2->getOpcode());
735 if (Commuted)
736 std::swap(MI1, MI2);
737
738 // 1. The previous instruction must be the same type as Inst.
739 // 2. The previous instruction must also be associative/commutative or be the
740 // inverse of such an operation (this can be different even for
741 // instructions with the same opcode if traits like fast-math-flags are
742 // included).
743 // 3. The previous instruction must have virtual register definitions for its
744 // operands in the same basic block as Inst.
745 // 4. The previous instruction's result must only be used by Inst.
746 return areOpcodesEqualOrInverse(Opcode, MI1->getOpcode()) &&
748 isAssociativeAndCommutative(*MI1, /* Invert */ true)) &&
750 MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg());
751}
752
753// 1. The operation must be associative and commutative or be the inverse of
754// such an operation.
755// 2. The instruction must have virtual register definitions for its
756// operands in the same basic block.
757// 3. The instruction must have a reassociable sibling.
759 bool &Commuted) const {
760 return (isAssociativeAndCommutative(Inst) ||
761 isAssociativeAndCommutative(Inst, /* Invert */ true)) &&
762 hasReassociableOperands(Inst, Inst.getParent()) &&
763 hasReassociableSibling(Inst, Commuted);
764}
765
766// The concept of the reassociation pass is that these operations can benefit
767// from this kind of transformation:
768//
769// A = ? op ?
770// B = A op X (Prev)
771// C = B op Y (Root)
772// -->
773// A = ? op ?
774// B = X op Y
775// C = A op B
776//
777// breaking the dependency between A and B, allowing them to be executed in
778// parallel (or back-to-back in a pipeline) instead of depending on each other.
779
780// FIXME: This has the potential to be expensive (compile time) while not
781// improving the code at all. Some ways to limit the overhead:
782// 1. Track successful transforms; bail out if hit rate gets too low.
783// 2. Only enable at -O3 or some other non-default optimization level.
784// 3. Pre-screen pattern candidates here: if an operand of the previous
785// instruction is known to not increase the critical path, then don't match
786// that pattern.
789 bool DoRegPressureReduce) const {
790 bool Commute;
791 if (isReassociationCandidate(Root, Commute)) {
792 // We found a sequence of instructions that may be suitable for a
793 // reassociation of operands to increase ILP. Specify each commutation
794 // possibility for the Prev instruction in the sequence and let the
795 // machine combiner decide if changing the operands is worthwhile.
796 if (Commute) {
799 } else {
802 }
803 return true;
804 }
805
806 return false;
807}
808
809/// Return true when a code sequence can improve loop throughput.
810bool
812 return false;
813}
814
815std::pair<unsigned, unsigned>
817 const MachineInstr &Root,
818 const MachineInstr &Prev) const {
819 bool AssocCommutRoot = isAssociativeAndCommutative(Root);
820 bool AssocCommutPrev = isAssociativeAndCommutative(Prev);
821
822 // Early exit if both opcodes are associative and commutative. It's a trivial
823 // reassociation when we only change operands order. In this case opcodes are
824 // not required to have inverse versions.
825 if (AssocCommutRoot && AssocCommutPrev) {
826 assert(Root.getOpcode() == Prev.getOpcode() && "Expected to be equal");
827 return std::make_pair(Root.getOpcode(), Root.getOpcode());
828 }
829
830 // At least one instruction is not associative or commutative.
831 // Since we have matched one of the reassociation patterns, we expect that the
832 // instructions' opcodes are equal or one of them is the inversion of the
833 // other.
835 "Incorrectly matched pattern");
836 unsigned AssocCommutOpcode = Root.getOpcode();
837 unsigned InverseOpcode = *getInverseOpcode(Root.getOpcode());
838 if (!AssocCommutRoot)
839 std::swap(AssocCommutOpcode, InverseOpcode);
840
841 // The transformation rule (`+` is any associative and commutative binary
842 // operation, `-` is the inverse):
843 // REASSOC_AX_BY:
844 // (A + X) + Y => A + (X + Y)
845 // (A + X) - Y => A + (X - Y)
846 // (A - X) + Y => A - (X - Y)
847 // (A - X) - Y => A - (X + Y)
848 // REASSOC_XA_BY:
849 // (X + A) + Y => (X + Y) + A
850 // (X + A) - Y => (X - Y) + A
851 // (X - A) + Y => (X + Y) - A
852 // (X - A) - Y => (X - Y) - A
853 // REASSOC_AX_YB:
854 // Y + (A + X) => (Y + X) + A
855 // Y - (A + X) => (Y - X) - A
856 // Y + (A - X) => (Y - X) + A
857 // Y - (A - X) => (Y + X) - A
858 // REASSOC_XA_YB:
859 // Y + (X + A) => (Y + X) + A
860 // Y - (X + A) => (Y - X) - A
861 // Y + (X - A) => (Y + X) - A
862 // Y - (X - A) => (Y - X) + A
863 switch (Pattern) {
864 default:
865 llvm_unreachable("Unexpected pattern");
867 if (!AssocCommutRoot && AssocCommutPrev)
868 return {AssocCommutOpcode, InverseOpcode};
869 if (AssocCommutRoot && !AssocCommutPrev)
870 return {InverseOpcode, InverseOpcode};
871 if (!AssocCommutRoot && !AssocCommutPrev)
872 return {InverseOpcode, AssocCommutOpcode};
873 break;
875 if (!AssocCommutRoot && AssocCommutPrev)
876 return {AssocCommutOpcode, InverseOpcode};
877 if (AssocCommutRoot && !AssocCommutPrev)
878 return {InverseOpcode, AssocCommutOpcode};
879 if (!AssocCommutRoot && !AssocCommutPrev)
880 return {InverseOpcode, InverseOpcode};
881 break;
883 if (!AssocCommutRoot && AssocCommutPrev)
884 return {InverseOpcode, InverseOpcode};
885 if (AssocCommutRoot && !AssocCommutPrev)
886 return {AssocCommutOpcode, InverseOpcode};
887 if (!AssocCommutRoot && !AssocCommutPrev)
888 return {InverseOpcode, AssocCommutOpcode};
889 break;
891 if (!AssocCommutRoot && AssocCommutPrev)
892 return {InverseOpcode, InverseOpcode};
893 if (AssocCommutRoot && !AssocCommutPrev)
894 return {InverseOpcode, AssocCommutOpcode};
895 if (!AssocCommutRoot && !AssocCommutPrev)
896 return {AssocCommutOpcode, InverseOpcode};
897 break;
898 }
899 llvm_unreachable("Unhandled combination");
900}
901
902// Return a pair of boolean flags showing if the new root and new prev operands
903// must be swapped. See visual example of the rule in
904// TargetInstrInfo::getReassociationOpcodes.
905static std::pair<bool, bool> mustSwapOperands(MachineCombinerPattern Pattern) {
906 switch (Pattern) {
907 default:
908 llvm_unreachable("Unexpected pattern");
910 return {false, false};
912 return {true, false};
914 return {true, true};
916 return {true, true};
917 }
918}
919
920/// Attempt the reassociation transformation to reduce critical path length.
921/// See the above comments before getMachineCombinerPatterns().
923 MachineInstr &Root, MachineInstr &Prev,
927 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
928 MachineFunction *MF = Root.getMF();
932 const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
933
934 // This array encodes the operand index for each parameter because the
935 // operands may be commuted. Each row corresponds to a pattern value,
936 // and each column specifies the index of A, B, X, Y.
937 unsigned OpIdx[4][4] = {
938 { 1, 1, 2, 2 },
939 { 1, 2, 2, 1 },
940 { 2, 1, 1, 2 },
941 { 2, 2, 1, 1 }
942 };
943
944 int Row;
945 switch (Pattern) {
946 case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break;
947 case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break;
948 case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break;
949 case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break;
950 default: llvm_unreachable("unexpected MachineCombinerPattern");
951 }
952
953 MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]);
954 MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]);
955 MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]);
956 MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]);
957 MachineOperand &OpC = Root.getOperand(0);
958
959 Register RegA = OpA.getReg();
960 Register RegB = OpB.getReg();
961 Register RegX = OpX.getReg();
962 Register RegY = OpY.getReg();
963 Register RegC = OpC.getReg();
964
965 if (RegA.isVirtual())
966 MRI.constrainRegClass(RegA, RC);
967 if (RegB.isVirtual())
968 MRI.constrainRegClass(RegB, RC);
969 if (RegX.isVirtual())
970 MRI.constrainRegClass(RegX, RC);
971 if (RegY.isVirtual())
972 MRI.constrainRegClass(RegY, RC);
973 if (RegC.isVirtual())
974 MRI.constrainRegClass(RegC, RC);
975
976 // Create a new virtual register for the result of (X op Y) instead of
977 // recycling RegB because the MachineCombiner's computation of the critical
978 // path requires a new register definition rather than an existing one.
979 Register NewVR = MRI.createVirtualRegister(RC);
980 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
981
982 auto [NewRootOpc, NewPrevOpc] = getReassociationOpcodes(Pattern, Root, Prev);
983 bool KillA = OpA.isKill();
984 bool KillX = OpX.isKill();
985 bool KillY = OpY.isKill();
986 bool KillNewVR = true;
987
988 auto [SwapRootOperands, SwapPrevOperands] = mustSwapOperands(Pattern);
989
990 if (SwapPrevOperands) {
991 std::swap(RegX, RegY);
992 std::swap(KillX, KillY);
993 }
994
995 // Create new instructions for insertion.
997 BuildMI(*MF, MIMetadata(Prev), TII->get(NewPrevOpc), NewVR)
998 .addReg(RegX, getKillRegState(KillX))
999 .addReg(RegY, getKillRegState(KillY))
1000 .setMIFlags(Prev.getFlags());
1001
1002 if (SwapRootOperands) {
1003 std::swap(RegA, NewVR);
1004 std::swap(KillA, KillNewVR);
1005 }
1006
1007 MachineInstrBuilder MIB2 =
1008 BuildMI(*MF, MIMetadata(Root), TII->get(NewRootOpc), RegC)
1009 .addReg(RegA, getKillRegState(KillA))
1010 .addReg(NewVR, getKillRegState(KillNewVR))
1011 .setMIFlags(Root.getFlags());
1012
1013 setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2);
1014
1015 // Record new instructions for insertion and old instructions for deletion.
1016 InsInstrs.push_back(MIB1);
1017 InsInstrs.push_back(MIB2);
1018 DelInstrs.push_back(&Prev);
1019 DelInstrs.push_back(&Root);
1020
1021 // We transformed:
1022 // B = A op X (Prev)
1023 // C = B op Y (Root)
1024 // Into:
1025 // B = X op Y (MIB1)
1026 // C = A op B (MIB2)
1027 // C has the same value as before, B doesn't; as such, keep the debug number
1028 // of C but not of B.
1029 if (unsigned OldRootNum = Root.peekDebugInstrNum())
1030 MIB2.getInstr()->setDebugInstrNum(OldRootNum);
1031}
1032
1037 DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const {
1039
1040 // Select the previous instruction in the sequence based on the input pattern.
1041 MachineInstr *Prev = nullptr;
1042 switch (Pattern) {
1045 Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
1046 break;
1049 Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
1050 break;
1051 default:
1052 llvm_unreachable("Unknown pattern for machine combiner");
1053 }
1054
1055 // Don't reassociate if Prev and Root are in different blocks.
1056 if (Prev->getParent() != Root.getParent())
1057 return;
1058
1059 reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
1060}
1061
1064}
1065
1066bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric(
1067 const MachineInstr &MI) const {
1068 const MachineFunction &MF = *MI.getMF();
1069 const MachineRegisterInfo &MRI = MF.getRegInfo();
1070
1071 // Remat clients assume operand 0 is the defined register.
1072 if (!MI.getNumOperands() || !MI.getOperand(0).isReg())
1073 return false;
1074 Register DefReg = MI.getOperand(0).getReg();
1075
1076 // A sub-register definition can only be rematerialized if the instruction
1077 // doesn't read the other parts of the register. Otherwise it is really a
1078 // read-modify-write operation on the full virtual register which cannot be
1079 // moved safely.
1080 if (DefReg.isVirtual() && MI.getOperand(0).getSubReg() &&
1081 MI.readsVirtualRegister(DefReg))
1082 return false;
1083
1084 // A load from a fixed stack slot can be rematerialized. This may be
1085 // redundant with subsequent checks, but it's target-independent,
1086 // simple, and a common case.
1087 int FrameIdx = 0;
1088 if (isLoadFromStackSlot(MI, FrameIdx) &&
1089 MF.getFrameInfo().isImmutableObjectIndex(FrameIdx))
1090 return true;
1091
1092 // Avoid instructions obviously unsafe for remat.
1093 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
1094 MI.hasUnmodeledSideEffects())
1095 return false;
1096
1097 // Don't remat inline asm. We have no idea how expensive it is
1098 // even if it's side effect free.
1099 if (MI.isInlineAsm())
1100 return false;
1101
1102 // Avoid instructions which load from potentially varying memory.
1103 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
1104 return false;
1105
1106 // If any of the registers accessed are non-constant, conservatively assume
1107 // the instruction is not rematerializable.
1108 for (const MachineOperand &MO : MI.operands()) {
1109 if (!MO.isReg()) continue;
1110 Register Reg = MO.getReg();
1111 if (Reg == 0)
1112 continue;
1113
1114 // Check for a well-behaved physical register.
1115 if (Reg.isPhysical()) {
1116 if (MO.isUse()) {
1117 // If the physreg has no defs anywhere, it's just an ambient register
1118 // and we can freely move its uses. Alternatively, if it's allocatable,
1119 // it could get allocated to something with a def during allocation.
1120 if (!MRI.isConstantPhysReg(Reg))
1121 return false;
1122 } else {
1123 // A physreg def. We can't remat it.
1124 return false;
1125 }
1126 continue;
1127 }
1128
1129 // Only allow one virtual-register def. There may be multiple defs of the
1130 // same virtual register, though.
1131 if (MO.isDef() && Reg != DefReg)
1132 return false;
1133
1134 // Don't allow any virtual-register uses. Rematting an instruction with
1135 // virtual register uses would length the live ranges of the uses, which
1136 // is not necessarily a good idea, certainly not "trivial".
1137 if (MO.isUse())
1138 return false;
1139 }
1140
1141 // Everything checked out.
1142 return true;
1143}
1144
1146 const MachineFunction *MF = MI.getMF();
1148 bool StackGrowsDown =
1150
1151 unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
1152 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
1153
1154 if (!isFrameInstr(MI))
1155 return 0;
1156
1157 int SPAdj = TFI->alignSPAdjust(getFrameSize(MI));
1158
1159 if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) ||
1160 (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode))
1161 SPAdj = -SPAdj;
1162
1163 return SPAdj;
1164}
1165
1166/// isSchedulingBoundary - Test if the given instruction should be
1167/// considered a scheduling boundary. This primarily includes labels
1168/// and terminators.
1170 const MachineBasicBlock *MBB,
1171 const MachineFunction &MF) const {
1172 // Terminators and labels can't be scheduled around.
1173 if (MI.isTerminator() || MI.isPosition())
1174 return true;
1175
1176 // INLINEASM_BR can jump to another block
1177 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
1178 return true;
1179
1180 // Don't attempt to schedule around any instruction that defines
1181 // a stack-oriented pointer, as it's unlikely to be profitable. This
1182 // saves compile time, because it doesn't require every single
1183 // stack slot reference to depend on the instruction that does the
1184 // modification.
1185 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
1187 return MI.modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI);
1188}
1189
1190// Provide a global flag for disabling the PreRA hazard recognizer that targets
1191// may choose to honor.
1194}
1195
1196// Default implementation of CreateTargetRAHazardRecognizer.
1199 const ScheduleDAG *DAG) const {
1200 // Dummy hazard recognizer allows all instructions to issue.
1201 return new ScheduleHazardRecognizer();
1202}
1203
1204// Default implementation of CreateTargetMIHazardRecognizer.
1206 const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
1207 return new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler");
1208}
1209
1210// Default implementation of CreateTargetPostRAHazardRecognizer.
1213 const ScheduleDAG *DAG) const {
1214 return new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched");
1215}
1216
1217// Default implementation of getMemOperandWithOffset.
1219 const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset,
1220 bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
1222 unsigned Width;
1223 if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable,
1224 Width, TRI) ||
1225 BaseOps.size() != 1)
1226 return false;
1227 BaseOp = BaseOps.front();
1228 return true;
1229}
1230
1231//===----------------------------------------------------------------------===//
1232// SelectionDAG latency interface.
1233//===----------------------------------------------------------------------===//
1234
1235int
1237 SDNode *DefNode, unsigned DefIdx,
1238 SDNode *UseNode, unsigned UseIdx) const {
1239 if (!ItinData || ItinData->isEmpty())
1240 return -1;
1241
1242 if (!DefNode->isMachineOpcode())
1243 return -1;
1244
1245 unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass();
1246 if (!UseNode->isMachineOpcode())
1247 return ItinData->getOperandCycle(DefClass, DefIdx);
1248 unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass();
1249 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1250}
1251
1253 SDNode *N) const {
1254 if (!ItinData || ItinData->isEmpty())
1255 return 1;
1256
1257 if (!N->isMachineOpcode())
1258 return 1;
1259
1260 return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass());
1261}
1262
1263//===----------------------------------------------------------------------===//
1264// MachineInstr latency interface.
1265//===----------------------------------------------------------------------===//
1266
1268 const MachineInstr &MI) const {
1269 if (!ItinData || ItinData->isEmpty())
1270 return 1;
1271
1272 unsigned Class = MI.getDesc().getSchedClass();
1273 int UOps = ItinData->Itineraries[Class].NumMicroOps;
1274 if (UOps >= 0)
1275 return UOps;
1276
1277 // The # of u-ops is dynamically determined. The specific target should
1278 // override this function to return the right number.
1279 return 1;
1280}
1281
1282/// Return the default expected latency for a def based on it's opcode.
1284 const MachineInstr &DefMI) const {
1285 if (DefMI.isTransient())
1286 return 0;
1287 if (DefMI.mayLoad())
1288 return SchedModel.LoadLatency;
1289 if (isHighLatencyDef(DefMI.getOpcode()))
1290 return SchedModel.HighLatency;
1291 return 1;
1292}
1293
1295 return 0;
1296}
1297
1299 const MachineInstr &MI,
1300 unsigned *PredCost) const {
1301 // Default to one cycle for no itinerary. However, an "empty" itinerary may
1302 // still have a MinLatency property, which getStageLatency checks.
1303 if (!ItinData)
1304 return MI.mayLoad() ? 2 : 1;
1305
1306 return ItinData->getStageLatency(MI.getDesc().getSchedClass());
1307}
1308
1310 const MachineInstr &DefMI,
1311 unsigned DefIdx) const {
1312 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
1313 if (!ItinData || ItinData->isEmpty())
1314 return false;
1315
1316 unsigned DefClass = DefMI.getDesc().getSchedClass();
1317 int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
1318 return (DefCycle != -1 && DefCycle <= 1);
1319}
1320
1321std::optional<ParamLoadedValue>
1323 Register Reg) const {
1324 const MachineFunction *MF = MI.getMF();
1327 int64_t Offset;
1328 bool OffsetIsScalable;
1329
1330 // To simplify the sub-register handling, verify that we only need to
1331 // consider physical registers.
1334
1335 if (auto DestSrc = isCopyInstr(MI)) {
1336 Register DestReg = DestSrc->Destination->getReg();
1337
1338 // If the copy destination is the forwarding reg, describe the forwarding
1339 // reg using the copy source as the backup location. Example:
1340 //
1341 // x0 = MOV x7
1342 // call callee(x0) ; x0 described as x7
1343 if (Reg == DestReg)
1344 return ParamLoadedValue(*DestSrc->Source, Expr);
1345
1346 // Cases where super- or sub-registers needs to be described should
1347 // be handled by the target's hook implementation.
1348 assert(!TRI->isSuperOrSubRegisterEq(Reg, DestReg) &&
1349 "TargetInstrInfo::describeLoadedValue can't describe super- or "
1350 "sub-regs for copy instructions");
1351 return std::nullopt;
1352 } else if (auto RegImm = isAddImmediate(MI, Reg)) {
1353 Register SrcReg = RegImm->Reg;
1354 Offset = RegImm->Imm;
1356 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
1357 } else if (MI.hasOneMemOperand()) {
1358 // Only describe memory which provably does not escape the function. As
1359 // described in llvm.org/PR43343, escaped memory may be clobbered by the
1360 // callee (or by another thread).
1361 const auto &TII = MF->getSubtarget().getInstrInfo();
1362 const MachineFrameInfo &MFI = MF->getFrameInfo();
1363 const MachineMemOperand *MMO = MI.memoperands()[0];
1364 const PseudoSourceValue *PSV = MMO->getPseudoValue();
1365
1366 // If the address points to "special" memory (e.g. a spill slot), it's
1367 // sufficient to check that it isn't aliased by any high-level IR value.
1368 if (!PSV || PSV->mayAlias(&MFI))
1369 return std::nullopt;
1370
1371 const MachineOperand *BaseOp;
1372 if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,
1373 TRI))
1374 return std::nullopt;
1375
1376 // FIXME: Scalable offsets are not yet handled in the offset code below.
1377 if (OffsetIsScalable)
1378 return std::nullopt;
1379
1380 // TODO: Can currently only handle mem instructions with a single define.
1381 // An example from the x86 target:
1382 // ...
1383 // DIV64m $rsp, 1, $noreg, 24, $noreg, implicit-def dead $rax, implicit-def $rdx
1384 // ...
1385 //
1386 if (MI.getNumExplicitDefs() != 1)
1387 return std::nullopt;
1388
1389 // TODO: In what way do we need to take Reg into consideration here?
1390
1393 Ops.push_back(dwarf::DW_OP_deref_size);
1394 Ops.push_back(MMO->getSize());
1395 Expr = DIExpression::prependOpcodes(Expr, Ops);
1396 return ParamLoadedValue(*BaseOp, Expr);
1397 }
1398
1399 return std::nullopt;
1400}
1401
1402/// Both DefMI and UseMI must be valid. By default, call directly to the
1403/// itinerary. This may be overriden by the target.
1405 const MachineInstr &DefMI,
1406 unsigned DefIdx,
1407 const MachineInstr &UseMI,
1408 unsigned UseIdx) const {
1409 unsigned DefClass = DefMI.getDesc().getSchedClass();
1410 unsigned UseClass = UseMI.getDesc().getSchedClass();
1411 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1412}
1413
1415 const MachineInstr &MI, unsigned DefIdx,
1416 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
1417 assert((MI.isRegSequence() ||
1418 MI.isRegSequenceLike()) && "Instruction do not have the proper type");
1419
1420 if (!MI.isRegSequence())
1421 return getRegSequenceLikeInputs(MI, DefIdx, InputRegs);
1422
1423 // We are looking at:
1424 // Def = REG_SEQUENCE v0, sub0, v1, sub1, ...
1425 assert(DefIdx == 0 && "REG_SEQUENCE only has one def");
1426 for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx;
1427 OpIdx += 2) {
1428 const MachineOperand &MOReg = MI.getOperand(OpIdx);
1429 if (MOReg.isUndef())
1430 continue;
1431 const MachineOperand &MOSubIdx = MI.getOperand(OpIdx + 1);
1432 assert(MOSubIdx.isImm() &&
1433 "One of the subindex of the reg_sequence is not an immediate");
1434 // Record Reg:SubReg, SubIdx.
1435 InputRegs.push_back(RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(),
1436 (unsigned)MOSubIdx.getImm()));
1437 }
1438 return true;
1439}
1440
1442 const MachineInstr &MI, unsigned DefIdx,
1443 RegSubRegPairAndIdx &InputReg) const {
1444 assert((MI.isExtractSubreg() ||
1445 MI.isExtractSubregLike()) && "Instruction do not have the proper type");
1446
1447 if (!MI.isExtractSubreg())
1448 return getExtractSubregLikeInputs(MI, DefIdx, InputReg);
1449
1450 // We are looking at:
1451 // Def = EXTRACT_SUBREG v0.sub1, sub0.
1452 assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def");
1453 const MachineOperand &MOReg = MI.getOperand(1);
1454 if (MOReg.isUndef())
1455 return false;
1456 const MachineOperand &MOSubIdx = MI.getOperand(2);
1457 assert(MOSubIdx.isImm() &&
1458 "The subindex of the extract_subreg is not an immediate");
1459
1460 InputReg.Reg = MOReg.getReg();
1461 InputReg.SubReg = MOReg.getSubReg();
1462 InputReg.SubIdx = (unsigned)MOSubIdx.getImm();
1463 return true;
1464}
1465
1467 const MachineInstr &MI, unsigned DefIdx,
1468 RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const {
1469 assert((MI.isInsertSubreg() ||
1470 MI.isInsertSubregLike()) && "Instruction do not have the proper type");
1471
1472 if (!MI.isInsertSubreg())
1473 return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg);
1474
1475 // We are looking at:
1476 // Def = INSERT_SEQUENCE v0, v1, sub0.
1477 assert(DefIdx == 0 && "INSERT_SUBREG only has one def");
1478 const MachineOperand &MOBaseReg = MI.getOperand(1);
1479 const MachineOperand &MOInsertedReg = MI.getOperand(2);
1480 if (MOInsertedReg.isUndef())
1481 return false;
1482 const MachineOperand &MOSubIdx = MI.getOperand(3);
1483 assert(MOSubIdx.isImm() &&
1484 "One of the subindex of the reg_sequence is not an immediate");
1485 BaseReg.Reg = MOBaseReg.getReg();
1486 BaseReg.SubReg = MOBaseReg.getSubReg();
1487
1488 InsertedReg.Reg = MOInsertedReg.getReg();
1489 InsertedReg.SubReg = MOInsertedReg.getSubReg();
1490 InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm();
1491 return true;
1492}
1493
1494// Returns a MIRPrinter comment for this machine operand.
1496 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
1497 const TargetRegisterInfo *TRI) const {
1498
1499 if (!MI.isInlineAsm())
1500 return "";
1501
1502 std::string Flags;
1504
1505 if (OpIdx == InlineAsm::MIOp_ExtraInfo) {
1506 // Print HasSideEffects, MayLoad, MayStore, IsAlignStack
1507 unsigned ExtraInfo = Op.getImm();
1508 bool First = true;
1509 for (StringRef Info : InlineAsm::getExtraInfoNames(ExtraInfo)) {
1510 if (!First)
1511 OS << " ";
1512 First = false;
1513 OS << Info;
1514 }
1515
1516 return OS.str();
1517 }
1518
1519 int FlagIdx = MI.findInlineAsmFlagIdx(OpIdx);
1520 if (FlagIdx < 0 || (unsigned)FlagIdx != OpIdx)
1521 return "";
1522
1523 assert(Op.isImm() && "Expected flag operand to be an immediate");
1524 // Pretty print the inline asm operand descriptor.
1525 unsigned Flag = Op.getImm();
1526 unsigned Kind = InlineAsm::getKind(Flag);
1527 OS << InlineAsm::getKindName(Kind);
1528
1529 unsigned RCID = 0;
1530 if (!InlineAsm::isImmKind(Flag) && !InlineAsm::isMemKind(Flag) &&
1532 if (TRI) {
1533 OS << ':' << TRI->getRegClassName(TRI->getRegClass(RCID));
1534 } else
1535 OS << ":RC" << RCID;
1536 }
1537
1538 if (InlineAsm::isMemKind(Flag)) {
1539 unsigned MCID = InlineAsm::getMemoryConstraintID(Flag);
1540 OS << ":" << InlineAsm::getMemConstraintName(MCID);
1541 }
1542
1543 unsigned TiedTo = 0;
1544 if (InlineAsm::isUseOperandTiedToDef(Flag, TiedTo))
1545 OS << " tiedto:$" << TiedTo;
1546
1547 return OS.str();
1548}
1549
1551
1553 Function &F, std::vector<outliner::Candidate> &Candidates) const {
1554 // Include target features from an arbitrary candidate for the outlined
1555 // function. This makes sure the outlined function knows what kinds of
1556 // instructions are going into it. This is fine, since all parent functions
1557 // must necessarily support the instructions that are in the outlined region.
1558 outliner::Candidate &FirstCand = Candidates.front();
1559 const Function &ParentFn = FirstCand.getMF()->getFunction();
1560 if (ParentFn.hasFnAttribute("target-features"))
1561 F.addFnAttr(ParentFn.getFnAttribute("target-features"));
1562 if (ParentFn.hasFnAttribute("target-cpu"))
1563 F.addFnAttr(ParentFn.getFnAttribute("target-cpu"));
1564
1565 // Set nounwind, so we don't generate eh_frame.
1566 if (llvm::all_of(Candidates, [](const outliner::Candidate &C) {
1567 return C.getMF()->getFunction().hasFnAttribute(Attribute::NoUnwind);
1568 }))
1569 F.addFnAttr(Attribute::NoUnwind);
1570}
1571
1573 MachineBasicBlock::iterator &MIT, unsigned Flags) const {
1574 MachineInstr &MI = *MIT;
1575
1576 // NOTE: MI.isMetaInstruction() will match CFI_INSTRUCTION, but some targets
1577 // have support for outlining those. Special-case that here.
1578 if (MI.isCFIInstruction())
1579 // Just go right to the target implementation.
1580 return getOutliningTypeImpl(MIT, Flags);
1581
1582 // Don't allow instructions that don't materialize to impact analysis.
1583 if (MI.isMetaInstruction())
1585
1586 // Be conservative about inline assembly.
1587 if (MI.isInlineAsm())
1589
1590 // Labels generally can't safely be outlined.
1591 if (MI.isLabel())
1593
1594 // Is this a terminator for a basic block?
1595 if (MI.isTerminator()) {
1596 // If this is a branch to another block, we can't outline it.
1597 if (!MI.getParent()->succ_empty())
1599
1600 // Don't outline if the branch is not unconditional.
1601 if (isPredicated(MI))
1603 }
1604
1605 // Make sure none of the operands of this instruction do anything that
1606 // might break if they're moved outside their current function.
1607 // This includes MachineBasicBlock references, BlockAddressses,
1608 // Constant pool indices and jump table indices.
1609 //
1610 // A quick note on MO_TargetIndex:
1611 // This doesn't seem to be used in any of the architectures that the
1612 // MachineOutliner supports, but it was still filtered out in all of them.
1613 // There was one exception (RISC-V), but MO_TargetIndex also isn't used there.
1614 // As such, this check is removed both here and in the target-specific
1615 // implementations. Instead, we assert to make sure this doesn't
1616 // catch anyone off-guard somewhere down the line.
1617 for (const MachineOperand &MOP : MI.operands()) {
1618 // If you hit this assertion, please remove it and adjust
1619 // `getOutliningTypeImpl` for your target appropriately if necessary.
1620 // Adding the assertion back to other supported architectures
1621 // would be nice too :)
1622 assert(!MOP.isTargetIndex() && "This isn't used quite yet!");
1623
1624 // CFI instructions should already have been filtered out at this point.
1625 assert(!MOP.isCFIIndex() && "CFI instructions handled elsewhere!");
1626
1627 // PrologEpilogInserter should've already run at this point.
1628 assert(!MOP.isFI() && "FrameIndex instructions should be gone by now!");
1629
1630 if (MOP.isMBB() || MOP.isBlockAddress() || MOP.isCPI() || MOP.isJTI())
1632 }
1633
1634 // If we don't know, delegate to the target-specific hook.
1635 return getOutliningTypeImpl(MIT, Flags);
1636}
1637
1639 unsigned &Flags) const {
1640 // Some instrumentations create special TargetOpcode at the start which
1641 // expands to special code sequences which must be present.
1642 auto First = MBB.getFirstNonDebugInstr();
1643 if (First != MBB.end() &&
1644 (First->getOpcode() == TargetOpcode::FENTRY_CALL ||
1645 First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER))
1646 return false;
1647
1648 return true;
1649}
unsigned SubReg
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains constants used for implementing Dwarf debug support.
uint64_t Size
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file contains some functions that are useful when dealing with strings.
static bool isAsmComment(const char *Str, const MCAsmInfo &MAI)
static const TargetRegisterClass * canFoldCopy(const MachineInstr &MI, unsigned FoldIdx)
static std::pair< bool, bool > mustSwapOperands(MachineCombinerPattern Pattern)
static cl::opt< bool > DisableHazardRecognizer("disable-sched-hazard", cl::Hidden, cl::init(false), cl::desc("Disable hazard detection during preRA scheduling"))
static MachineInstr * foldPatchpoint(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, int FrameIndex, const TargetInstrInfo &TII)
This file describes how to lower LLVM code to machine code.
@ Flags
Definition: TextStubV5.cpp:93
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
DWARF expression.
static void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:238
A debug info location.
Definition: DebugLoc.h:33
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:670
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:319
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:644
static bool isMemKind(unsigned Flag)
Definition: InlineAsm.h:301
static StringRef getKindName(unsigned Kind)
Definition: InlineAsm.h:414
static std::vector< StringRef > getExtraInfoNames(unsigned ExtraInfo)
Definition: InlineAsm.h:390
static StringRef getMemConstraintName(unsigned Constraint)
Definition: InlineAsm.h:434
static unsigned getMemoryConstraintID(unsigned Flag)
Definition: InlineAsm.h:355
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:369
static bool isImmKind(unsigned Flag)
Definition: InlineAsm.h:300
static bool hasRegClassConstraint(unsigned Flag, unsigned &RC)
hasRegClassConstraint - Returns true if the flag contains a register class constraint.
Definition: InlineAsm.h:378
static unsigned getKind(unsigned Flags)
Definition: InlineAsm.h:351
Itinerary data supplied by a subtarget to be used by a target.
int getOperandLatency(unsigned DefClass, unsigned DefIdx, unsigned UseClass, unsigned UseIdx) const
Compute and return the use operand latency of a given itinerary class and operand index if the value ...
unsigned getStageLatency(unsigned ItinClassIndx) const
Return the total stage latency of the given class.
int getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
const InstrItinerary * Itineraries
Array of itineraries selected.
bool isEmpty() const
Returns true if there are no itineraries.
This class is intended to be used as a base class for asm properties and features specific to the tar...
Definition: MCAsmInfo.h:56
virtual unsigned getMaxInstLength(const MCSubtargetInfo *STI=nullptr) const
Returns the maximum possible encoded instruction size in bytes.
Definition: MCAsmInfo.h:641
StringRef getCommentString() const
Definition: MCAsmInfo.h:655
const char * getSeparatorString() const
Definition: MCAsmInfo.h:649
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:184
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getSchedClass() const
Return the scheduling class for this instruction.
Definition: MCInstrDesc.h:596
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:237
ArrayRef< MCOperandInfo > operands() const
Definition: MCInstrDesc.h:239
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:247
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z,...
Definition: MCInstrDesc.h:480
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition: MCInstrInfo.h:63
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1399
Set of metadata that should be preserved when using BuildMI().
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator getFirstNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the first non-debug instruction in the basic block, or end().
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isImmutableObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to an immutable object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool hasProperty(Property P) const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineInstr & cloneMachineInstrBundle(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig)
Clones instruction or the whole instruction bundle Orig and insert into MBB before InsertBefore.
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineFunctionProperties & getProperties() const
Get the function properties.
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
void eraseCallSiteInfo(const MachineInstr *MI)
Following functions update call site info.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:516
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:313
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:519
void setDebugInstrNum(unsigned Num)
Set instruction number of this MachineInstr.
Definition: MachineInstr.h:496
mmo_iterator memoperands_end() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:738
unsigned peekDebugInstrNum() const
Examine the instruction number of this MachineInstr.
Definition: MachineInstr.h:492
void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)
Assign this MachineInstr's memory reference descriptor list.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool isNotDuplicable(QueryType Type=AnyInBundle) const
Return true if this instruction cannot be safely duplicated.
Definition: MachineInstr.h:970
void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:731
void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI)
Clone another MachineInstr's pre- and post- instruction symbols and replace ours with it.
bool isIdenticalTo(const MachineInstr &Other, MICheckType Check=CheckDefs) const
Return true if this instruction is identical to Other.
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:713
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:352
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
bool canFoldAsLoad(QueryType Type=IgnoreBundle) const
Return true for instructions that can be folded as memory operands in other instructions.
const TargetRegisterClass * getRegClassConstraint(unsigned OpIdx, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const
Compute the static register class constraint for operand OpIdx.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
uint64_t getSize() const
Return the size in bytes of the memory reference.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setIsInternalRead(bool Val=true)
void setImm(int64_t immVal)
int64_t getImm() const
void setIsRenamable(bool Val=true)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
void setMBB(MachineBasicBlock *MBB)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
MI-level patchpoint operands.
Definition: StackMaps.h:76
Special value supplied for machine level alias analysis.
virtual bool mayAlias(const MachineFrameInfo *) const
Return true if the memory pointed to by this PseudoSourceValue can ever alias an LLVM IR Value.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:97
bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
HazardRecognizer - This determines whether or not an instruction can be issued this cycle,...
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
MI-level stackmap operands.
Definition: StackMaps.h:35
MI-level Statepoint operands.
Definition: StackMaps.h:158
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
Information about stack frame layout on the target.
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
int alignSPAdjust(int SPAdj) const
alignSPAdjust - This method aligns the stack adjustment to the correct alignment.
TargetInstrInfo - Interface to description of machine instruction set.
virtual ScheduleHazardRecognizer * CreateTargetPostRAHazardRecognizer(const InstrItineraryData *, const ScheduleDAG *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
virtual MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const
Target-dependent implementation for foldMemoryOperand.
virtual bool hasLowDefLatency(const TargetSchedModel &SchedModel, const MachineInstr &DefMI, unsigned DefIdx) const
Compute operand latency of a def of 'Reg'.
virtual void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, MachineInstr &NewMI1, MachineInstr &NewMI2) const
This is an architecture-specific helper function of reassociateOps.
virtual bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, const TargetRegisterInfo *TRI) const
Get zero or more base operands and the byte offset of an instruction that reads/writes memory.
virtual unsigned getNumMicroOps(const InstrItineraryData *ItinData, const MachineInstr &MI) const
Return the number of u-operations the given machine instruction will be decoded to on the target cpu.
virtual int getSPAdjust(const MachineInstr &MI) const
Returns the actual stack pointer adjustment made by an instruction as part of a call sequence.
virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const
Load the specified register of the given register class from the specified stack frame index.
virtual void ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, MachineBasicBlock *NewDest) const
Delete the instruction OldInst and everything after it, replacing it with an unconditional branch to ...
virtual bool PredicateInstruction(MachineInstr &MI, ArrayRef< MachineOperand > Pred) const
Convert the instruction into a predicated instruction.
bool areOpcodesEqualOrInverse(unsigned Opcode1, unsigned Opcode2) const
Return true when \P Opcode1 or its inversion is equal to \P Opcode2.
virtual bool getInsertSubregLikeInputs(const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const
Target-dependent implementation of getInsertSubregInputs.
virtual std::pair< unsigned, unsigned > getPatchpointUnfoldableRange(const MachineInstr &MI) const
For a patchpoint, stackmap, or statepoint intrinsic, return the range of operands which can't be fold...
outliner::InstrType getOutliningType(MachineBasicBlock::iterator &MIT, unsigned Flags) const
Returns how or if MIT should be outlined.
virtual int getOperandLatency(const InstrItineraryData *ItinData, SDNode *DefNode, unsigned DefIdx, SDNode *UseNode, unsigned UseIdx) const
virtual void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const
Store the specified register of the given register class to the specified stack frame index.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual void mergeOutliningCandidateAttributes(Function &F, std::vector< outliner::Candidate > &Candidates) const
Optional target hook to create the LLVM IR attributes for the outlined function.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool isUnpredicatedTerminator(const MachineInstr &MI) const
Returns true if the instruction is a terminator instruction that has not been predicated.
virtual void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const
Insert a noop into the instruction stream at the specified point.
bool isFrameInstr(const MachineInstr &I) const
Returns true if the argument is a frame pseudo instruction.
virtual bool getRegSequenceLikeInputs(const MachineInstr &MI, unsigned DefIdx, SmallVectorImpl< RegSubRegPairAndIdx > &InputRegs) const
Target-dependent implementation of getRegSequenceInputs.
virtual bool getStackSlotRange(const TargetRegisterClass *RC, unsigned SubIdx, unsigned &Size, unsigned &Offset, const MachineFunction &MF) const
Compute the size in bytes and offset within a stack slot of a spilled register or subregister.
virtual ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *, const ScheduleDAGMI *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
virtual bool hasStoreToStackSlot(const MachineInstr &MI, SmallVectorImpl< const MachineMemOperand * > &Accesses) const
If the specified machine instruction has a store to a stack slot, return true along with the FrameInd...
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual unsigned getInlineAsmLength(const char *Str, const MCAsmInfo &MAI, const TargetSubtargetInfo *STI=nullptr) const
Measure the specified inline asm to determine an approximation of its length.
virtual outliner::InstrType getOutliningTypeImpl(MachineBasicBlock::iterator &MIT, unsigned Flags) const
Target-dependent implementation for getOutliningTypeImpl.
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
MachineInstr * foldMemoryOperand(MachineInstr &MI, ArrayRef< unsigned > Ops, int FI, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const
Attempt to fold a load or store of the specified stack slot into the specified machine instruction fo...
virtual ScheduleHazardRecognizer * CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const
Insert branch code into the end of the specified MachineBasicBlock.
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise).
virtual MCInst getNop() const
Return the noop instruction to use for a noop.
virtual MachineInstr & duplicate(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const
Clones instruction or the whole instruction bundle Orig and insert into MBB before InsertBefore.
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const
Optional target hook that returns true if MBB is safe to outline from, and returns any target-specifi...
MachineInstr * commuteInstruction(MachineInstr &MI, bool NewMI=false, unsigned OpIdx1=CommuteAnyOperandIndex, unsigned OpIdx2=CommuteAnyOperandIndex) const
This method commutes the operands of the given machine instruction MI.
virtual void genAlternativeCodeSequence(MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert=false) const
Return true when \P Inst is both associative and commutative.
virtual void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, const TargetRegisterInfo &TRI) const
Re-issue the specified 'original' instruction at the specific location targeting a new destination re...
virtual std::optional< unsigned > getInverseOpcode(unsigned Opcode) const
Return the inverse operation opcode if it exists for \P Opcode (e.g.
virtual void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned Quantity) const
Insert noops into the instruction stream at the specified point.
unsigned getCallFrameDestroyOpcode() const
int64_t getFrameSize(const MachineInstr &I) const
Returns size of the frame associated with the given frame instruction.
virtual bool isPredicated(const MachineInstr &MI) const
Returns true if the instruction is already predicated.
bool getInsertSubregInputs(const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const
Build the equivalent inputs of a INSERT_SUBREG for the given MI and DefIdx.
virtual bool isThroughputPattern(MachineCombinerPattern Pattern) const
Return true when a code sequence can improve throughput.
virtual ~TargetInstrInfo()
virtual unsigned getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr &MI, unsigned *PredCost=nullptr) const
Compute the instruction latency of a given instruction.
virtual bool produceSameValue(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo *MRI=nullptr) const
Return true if two machine instructions would produce identical values.
std::optional< DestSourcePair > isCopyInstr(const MachineInstr &MI) const
If the specific machine instruction is a instruction that moves/copies value from one register to ano...
bool isReassociationCandidate(const MachineInstr &Inst, bool &Commuted) const
Return true if the input \P Inst is part of a chain of dependent ops that are suitable for reassociat...
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
std::pair< unsigned, unsigned > getReassociationOpcodes(MachineCombinerPattern Pattern, const MachineInstr &Root, const MachineInstr &Prev) const
Reassociation of some instructions requires inverse operations (e.g.
virtual unsigned getPredicationCost(const MachineInstr &MI) const
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual MachineTraceStrategy getMachineCombinerTraceStrategy() const
Return a strategy that MachineCombiner must use when creating traces.
bool getRegSequenceInputs(const MachineInstr &MI, unsigned DefIdx, SmallVectorImpl< RegSubRegPairAndIdx > &InputRegs) const
Build the equivalent inputs of a REG_SEQUENCE for the given MI and DefIdx.
virtual bool hasLoadFromStackSlot(const MachineInstr &MI, SmallVectorImpl< const MachineMemOperand * > &Accesses) const
If the specified machine instruction has a load from a stack slot, return true along with the FrameIn...
virtual std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const
If the specific machine instruction is an instruction that adds an immediate value and a physical reg...
unsigned defaultDefLatency(const MCSchedModel &SchedModel, const MachineInstr &DefMI) const
Return the default expected latency for a def based on its opcode.
static const unsigned CommuteAnyOperandIndex
virtual bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const
Return true when \P Inst has reassociable sibling.
virtual std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const
void reassociateOps(MachineInstr &Root, MachineInstr &Prev, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const
Attempt to reassociate \P Root and \P Prev according to \P Pattern to reduce critical path length.
virtual bool isHighLatencyDef(int opc) const
Return true if this opcode has high latency to its result.
static bool fixCommutedOpIndices(unsigned &ResultIdx1, unsigned &ResultIdx2, unsigned CommutableOpIdx1, unsigned CommutableOpIdx2)
Assigns the (CommutableOpIdx1, CommutableOpIdx2) pair of commutable operand indices to (ResultIdx1,...
virtual unsigned isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const
If the specified machine instruction is a direct load from a stack slot, return the virtual or physic...
bool getExtractSubregInputs(const MachineInstr &MI, unsigned DefIdx, RegSubRegPairAndIdx &InputReg) const
Build the equivalent inputs of a EXTRACT_SUBREG for the given MI and DefIdx.
virtual bool getExtractSubregLikeInputs(const MachineInstr &MI, unsigned DefIdx, RegSubRegPairAndIdx &InputReg) const
Target-dependent implementation of getExtractSubregInputs.
bool usePreRAHazardRecognizer() const
Provide a global flag for disabling the PreRA hazard recognizer that targets may choose to honor.
virtual const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum,...
bool getMemOperandWithOffset(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const
Get the base operand and byte offset of an instruction that reads/writes memory.
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Provide an instruction scheduling machine model to CodeGen passes.
const InstrItineraryData * getInstrItineraries() const
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition: CallingConv.h:76
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
@ Length
Definition: DWP.cpp:406
MachineTraceStrategy
Strategies for selecting traces.
@ TS_MinInstrCount
Select the trace through a block that has the fewest instructions.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1782
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
MachineCombinerPattern
These are instruction patterns matched by the machine combiner pass.
unsigned getKillRegState(bool B)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1939
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
#define N
Machine model for scheduling, bundling, and heuristics.
Definition: MCSchedule.h:244
unsigned LoadLatency
Definition: MCSchedule.h:285
unsigned HighLatency
Definition: MCSchedule.h:292
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
A pair composed of a pair of a register and a sub-register index, and another sub-register index.
A pair composed of a register and a sub-register index.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineBasicBlock::iterator & front()
MachineFunction * getMF() const