Line data Source code
1 : //===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // Pass to verify generated machine code. The following is checked:
11 : //
12 : // Operand counts: All explicit operands must be present.
13 : //
14 : // Register classes: All physical and virtual register operands must be
15 : // compatible with the register class required by the instruction descriptor.
16 : //
17 : // Register live intervals: Registers must be defined only once, and must be
18 : // defined before use.
19 : //
20 : // The machine code verifier is enabled from LLVMTargetMachine.cpp with the
21 : // command-line option -verify-machineinstrs, or by defining the environment
22 : // variable LLVM_VERIFY_MACHINEINSTRS to the name of a file that will receive
23 : // the verifier errors.
24 : //===----------------------------------------------------------------------===//
25 :
26 : #include "LiveRangeCalc.h"
27 : #include "llvm/ADT/BitVector.h"
28 : #include "llvm/ADT/DenseMap.h"
29 : #include "llvm/ADT/DenseSet.h"
30 : #include "llvm/ADT/DepthFirstIterator.h"
31 : #include "llvm/ADT/STLExtras.h"
32 : #include "llvm/ADT/SetOperations.h"
33 : #include "llvm/ADT/SmallPtrSet.h"
34 : #include "llvm/ADT/SmallVector.h"
35 : #include "llvm/ADT/StringRef.h"
36 : #include "llvm/ADT/Twine.h"
37 : #include "llvm/Analysis/EHPersonalities.h"
38 : #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
39 : #include "llvm/CodeGen/LiveInterval.h"
40 : #include "llvm/CodeGen/LiveIntervals.h"
41 : #include "llvm/CodeGen/LiveStacks.h"
42 : #include "llvm/CodeGen/LiveVariables.h"
43 : #include "llvm/CodeGen/MachineBasicBlock.h"
44 : #include "llvm/CodeGen/MachineFrameInfo.h"
45 : #include "llvm/CodeGen/MachineFunction.h"
46 : #include "llvm/CodeGen/MachineFunctionPass.h"
47 : #include "llvm/CodeGen/MachineInstr.h"
48 : #include "llvm/CodeGen/MachineInstrBundle.h"
49 : #include "llvm/CodeGen/MachineMemOperand.h"
50 : #include "llvm/CodeGen/MachineOperand.h"
51 : #include "llvm/CodeGen/MachineRegisterInfo.h"
52 : #include "llvm/CodeGen/PseudoSourceValue.h"
53 : #include "llvm/CodeGen/SlotIndexes.h"
54 : #include "llvm/CodeGen/StackMaps.h"
55 : #include "llvm/CodeGen/TargetInstrInfo.h"
56 : #include "llvm/CodeGen/TargetOpcodes.h"
57 : #include "llvm/CodeGen/TargetRegisterInfo.h"
58 : #include "llvm/CodeGen/TargetSubtargetInfo.h"
59 : #include "llvm/IR/BasicBlock.h"
60 : #include "llvm/IR/Function.h"
61 : #include "llvm/IR/InlineAsm.h"
62 : #include "llvm/IR/Instructions.h"
63 : #include "llvm/MC/LaneBitmask.h"
64 : #include "llvm/MC/MCAsmInfo.h"
65 : #include "llvm/MC/MCInstrDesc.h"
66 : #include "llvm/MC/MCRegisterInfo.h"
67 : #include "llvm/MC/MCTargetOptions.h"
68 : #include "llvm/Pass.h"
69 : #include "llvm/Support/Casting.h"
70 : #include "llvm/Support/ErrorHandling.h"
71 : #include "llvm/Support/LowLevelTypeImpl.h"
72 : #include "llvm/Support/MathExtras.h"
73 : #include "llvm/Support/raw_ostream.h"
74 : #include "llvm/Target/TargetMachine.h"
75 : #include <algorithm>
76 : #include <cassert>
77 : #include <cstddef>
78 : #include <cstdint>
79 : #include <iterator>
80 : #include <string>
81 : #include <utility>
82 :
83 : using namespace llvm;
84 :
85 : namespace {
86 :
87 : struct MachineVerifier {
88 1662010 : MachineVerifier(Pass *pass, const char *b) : PASS(pass), Banner(b) {}
89 :
90 : unsigned verify(MachineFunction &MF);
91 :
92 : Pass *const PASS;
93 : const char *Banner;
94 : const MachineFunction *MF;
95 : const TargetMachine *TM;
96 : const TargetInstrInfo *TII;
97 : const TargetRegisterInfo *TRI;
98 : const MachineRegisterInfo *MRI;
99 :
100 : unsigned foundErrors;
101 :
102 : // Avoid querying the MachineFunctionProperties for each operand.
103 : bool isFunctionRegBankSelected;
104 : bool isFunctionSelected;
105 :
106 : using RegVector = SmallVector<unsigned, 16>;
107 : using RegMaskVector = SmallVector<const uint32_t *, 4>;
108 : using RegSet = DenseSet<unsigned>;
109 : using RegMap = DenseMap<unsigned, const MachineInstr *>;
110 : using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;
111 :
112 : const MachineInstr *FirstNonPHI;
113 : const MachineInstr *FirstTerminator;
114 : BlockSet FunctionBlocks;
115 :
116 : BitVector regsReserved;
117 : RegSet regsLive;
118 : RegVector regsDefined, regsDead, regsKilled;
119 : RegMaskVector regMasks;
120 :
121 : SlotIndex lastIndex;
122 :
123 : // Add Reg and any sub-registers to RV
124 0 : void addRegWithSubRegs(RegVector &RV, unsigned Reg) {
125 0 : RV.push_back(Reg);
126 0 : if (TargetRegisterInfo::isPhysicalRegister(Reg))
127 0 : for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs)
128 0 : RV.push_back(*SubRegs);
129 0 : }
130 :
131 : struct BBInfo {
132 : // Is this MBB reachable from the MF entry point?
133 : bool reachable = false;
134 :
135 : // Vregs that must be live in because they are used without being
136 : // defined. Map value is the user.
137 : RegMap vregsLiveIn;
138 :
139 : // Regs killed in MBB. They may be defined again, and will then be in both
140 : // regsKilled and regsLiveOut.
141 : RegSet regsKilled;
142 :
143 : // Regs defined in MBB and live out. Note that vregs passing through may
144 : // be live out without being mentioned here.
145 : RegSet regsLiveOut;
146 :
147 : // Vregs that pass through MBB untouched. This set is disjoint from
148 : // regsKilled and regsLiveOut.
149 : RegSet vregsPassed;
150 :
151 : // Vregs that must pass through MBB because they are needed by a successor
152 : // block. This set is disjoint from regsLiveOut.
153 : RegSet vregsRequired;
154 :
155 : // Set versions of block's predecessor and successor lists.
156 : BlockSet Preds, Succs;
157 :
158 2091781 : BBInfo() = default;
159 :
160 : // Add register to vregsPassed if it belongs there. Return true if
161 : // anything changed.
162 18443045 : bool addPassed(unsigned Reg) {
163 36886090 : if (!TargetRegisterInfo::isVirtualRegister(Reg))
164 : return false;
165 : if (regsKilled.count(Reg) || regsLiveOut.count(Reg))
166 356704 : return false;
167 : return vregsPassed.insert(Reg).second;
168 : }
169 :
170 : // Same for a full set.
171 836798 : bool addPassed(const RegSet &RS) {
172 : bool changed = false;
173 19279843 : for (RegSet::const_iterator I = RS.begin(), E = RS.end(); I != E; ++I)
174 18443045 : if (addPassed(*I))
175 : changed = true;
176 836798 : return changed;
177 : }
178 :
179 : // Add register to vregsRequired if it belongs there. Return true if
180 : // anything changed.
181 1189646 : bool addRequired(unsigned Reg) {
182 2379292 : if (!TargetRegisterInfo::isVirtualRegister(Reg))
183 : return false;
184 : if (regsLiveOut.count(Reg))
185 474433 : return false;
186 : return vregsRequired.insert(Reg).second;
187 : }
188 :
189 : // Same for a full set.
190 121837 : bool addRequired(const RegSet &RS) {
191 : bool changed = false;
192 920749 : for (RegSet::const_iterator I = RS.begin(), E = RS.end(); I != E; ++I)
193 798912 : if (addRequired(*I))
194 : changed = true;
195 121837 : return changed;
196 : }
197 :
198 : // Same for a full map.
199 613211 : bool addRequired(const RegMap &RM) {
200 : bool changed = false;
201 1003945 : for (RegMap::const_iterator I = RM.begin(), E = RM.end(); I != E; ++I)
202 390734 : if (addRequired(I->first))
203 : changed = true;
204 613211 : return changed;
205 : }
206 :
207 : // Live-out registers are either in regsLiveOut or vregsPassed.
208 171505 : bool isLiveOut(unsigned Reg) const {
209 171505 : return regsLiveOut.count(Reg) || vregsPassed.count(Reg);
210 : }
211 : };
212 :
213 : // Extra register info per MBB.
214 : DenseMap<const MachineBasicBlock*, BBInfo> MBBInfoMap;
215 :
216 : bool isReserved(unsigned Reg) {
217 12836614 : return Reg < regsReserved.size() && regsReserved.test(Reg);
218 : }
219 :
220 1132468 : bool isAllocatable(unsigned Reg) const {
221 1132468 : return Reg < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) &&
222 1132468 : !regsReserved.test(Reg);
223 : }
224 :
225 : // Analysis information if available
226 : LiveVariables *LiveVars;
227 : LiveIntervals *LiveInts;
228 : LiveStacks *LiveStks;
229 : SlotIndexes *Indexes;
230 :
231 : void visitMachineFunctionBefore();
232 : void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
233 : void visitMachineBundleBefore(const MachineInstr *MI);
234 : void visitMachineInstrBefore(const MachineInstr *MI);
235 : void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
236 : void visitMachineInstrAfter(const MachineInstr *MI);
237 : void visitMachineBundleAfter(const MachineInstr *MI);
238 : void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);
239 : void visitMachineFunctionAfter();
240 :
241 : void report(const char *msg, const MachineFunction *MF);
242 : void report(const char *msg, const MachineBasicBlock *MBB);
243 : void report(const char *msg, const MachineInstr *MI);
244 : void report(const char *msg, const MachineOperand *MO, unsigned MONum,
245 : LLT MOVRegType = LLT{});
246 :
247 : void report_context(const LiveInterval &LI) const;
248 : void report_context(const LiveRange &LR, unsigned VRegUnit,
249 : LaneBitmask LaneMask) const;
250 : void report_context(const LiveRange::Segment &S) const;
251 : void report_context(const VNInfo &VNI) const;
252 : void report_context(SlotIndex Pos) const;
253 : void report_context_liverange(const LiveRange &LR) const;
254 : void report_context_lanemask(LaneBitmask LaneMask) const;
255 : void report_context_vreg(unsigned VReg) const;
256 : void report_context_vreg_regunit(unsigned VRegOrUnit) const;
257 :
258 : void verifyInlineAsm(const MachineInstr *MI);
259 :
260 : void checkLiveness(const MachineOperand *MO, unsigned MONum);
261 : void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
262 : SlotIndex UseIdx, const LiveRange &LR, unsigned VRegOrUnit,
263 : LaneBitmask LaneMask = LaneBitmask::getNone());
264 : void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
265 : SlotIndex DefIdx, const LiveRange &LR, unsigned VRegOrUnit,
266 : bool SubRangeCheck = false,
267 : LaneBitmask LaneMask = LaneBitmask::getNone());
268 :
269 : void markReachable(const MachineBasicBlock *MBB);
270 : void calcRegsPassed();
271 : void checkPHIOps(const MachineBasicBlock &MBB);
272 :
273 : void calcRegsRequired();
274 : void verifyLiveVariables();
275 : void verifyLiveIntervals();
276 : void verifyLiveInterval(const LiveInterval&);
277 : void verifyLiveRangeValue(const LiveRange&, const VNInfo*, unsigned,
278 : LaneBitmask);
279 : void verifyLiveRangeSegment(const LiveRange&,
280 : const LiveRange::const_iterator I, unsigned,
281 : LaneBitmask);
282 : void verifyLiveRange(const LiveRange&, unsigned,
283 : LaneBitmask LaneMask = LaneBitmask::getNone());
284 :
285 : void verifyStackFrame();
286 :
287 : void verifySlotIndexes() const;
288 : void verifyProperties(const MachineFunction &MF);
289 : };
290 :
291 : struct MachineVerifierPass : public MachineFunctionPass {
292 : static char ID; // Pass ID, replacement for typeid
293 :
294 : const std::string Banner;
295 :
296 166085 : MachineVerifierPass(std::string banner = std::string())
297 166085 : : MachineFunctionPass(ID), Banner(std::move(banner)) {
298 166085 : initializeMachineVerifierPassPass(*PassRegistry::getPassRegistry());
299 166085 : }
300 :
301 166085 : void getAnalysisUsage(AnalysisUsage &AU) const override {
302 : AU.setPreservesAll();
303 166085 : MachineFunctionPass::getAnalysisUsage(AU);
304 166085 : }
305 :
306 1657362 : bool runOnMachineFunction(MachineFunction &MF) override {
307 1657362 : unsigned FoundErrors = MachineVerifier(this, Banner.c_str()).verify(MF);
308 1657362 : if (FoundErrors)
309 3 : report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors.");
310 1657359 : return false;
311 : }
312 : };
313 :
314 : } // end anonymous namespace
315 :
316 : char MachineVerifierPass::ID = 0;
317 :
318 251232 : INITIALIZE_PASS(MachineVerifierPass, "machineverifier",
319 : "Verify generated machine code", false, false)
320 :
321 166083 : FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {
322 166083 : return new MachineVerifierPass(Banner);
323 : }
324 :
325 4648 : bool MachineFunction::verify(Pass *p, const char *Banner, bool AbortOnErrors)
326 : const {
327 : MachineFunction &MF = const_cast<MachineFunction&>(*this);
328 4648 : unsigned FoundErrors = MachineVerifier(p, Banner).verify(MF);
329 4648 : if (AbortOnErrors && FoundErrors)
330 38 : report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors.");
331 4610 : return FoundErrors == 0;
332 : }
333 :
334 0 : void MachineVerifier::verifySlotIndexes() const {
335 : if (Indexes == nullptr)
336 0 : return;
337 :
338 : // Ensure the IdxMBB list is sorted by slot indexes.
339 : SlotIndex Last;
340 : for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),
341 : E = Indexes->MBBIndexEnd(); I != E; ++I) {
342 : assert(!Last.isValid() || I->first > Last);
343 : Last = I->first;
344 : }
345 : }
346 :
347 1661447 : void MachineVerifier::verifyProperties(const MachineFunction &MF) {
348 : // If a pass has introduced virtual registers without clearing the
349 : // NoVRegs property (or set it without allocating the vregs)
350 : // then report an error.
351 : if (MF.getProperties().hasProperty(
352 1661447 : MachineFunctionProperties::Property::NoVRegs) &&
353 835182 : MRI->getNumVirtRegs())
354 0 : report("Function has NoVRegs property but there are VReg operands", &MF);
355 1661447 : }
356 :
357 1662010 : unsigned MachineVerifier::verify(MachineFunction &MF) {
358 1662010 : foundErrors = 0;
359 :
360 1662010 : this->MF = &MF;
361 1662010 : TM = &MF.getTarget();
362 1662010 : TII = MF.getSubtarget().getInstrInfo();
363 1662010 : TRI = MF.getSubtarget().getRegisterInfo();
364 1662010 : MRI = &MF.getRegInfo();
365 :
366 : const bool isFunctionFailedISel = MF.getProperties().hasProperty(
367 : MachineFunctionProperties::Property::FailedISel);
368 :
369 : // If we're mid-GlobalISel and we already triggered the fallback path then
370 : // it's expected that the MIR is somewhat broken but that's ok since we'll
371 : // reset it and clear the FailedISel attribute in ResetMachineFunctions.
372 1662010 : if (isFunctionFailedISel)
373 563 : return foundErrors;
374 :
375 1661447 : isFunctionRegBankSelected =
376 1661447 : !isFunctionFailedISel &&
377 : MF.getProperties().hasProperty(
378 : MachineFunctionProperties::Property::RegBankSelected);
379 1661447 : isFunctionSelected = !isFunctionFailedISel &&
380 : MF.getProperties().hasProperty(
381 : MachineFunctionProperties::Property::Selected);
382 1661447 : LiveVars = nullptr;
383 1661447 : LiveInts = nullptr;
384 1661447 : LiveStks = nullptr;
385 1661447 : Indexes = nullptr;
386 1661447 : if (PASS) {
387 1656983 : LiveInts = PASS->getAnalysisIfAvailable<LiveIntervals>();
388 : // We don't want to verify LiveVariables if LiveIntervals is available.
389 1656983 : if (!LiveInts)
390 1459574 : LiveVars = PASS->getAnalysisIfAvailable<LiveVariables>();
391 1656983 : LiveStks = PASS->getAnalysisIfAvailable<LiveStacks>();
392 1656983 : Indexes = PASS->getAnalysisIfAvailable<SlotIndexes>();
393 : }
394 :
395 : verifySlotIndexes();
396 :
397 1661447 : verifyProperties(MF);
398 :
399 1661447 : visitMachineFunctionBefore();
400 : for (MachineFunction::const_iterator MFI = MF.begin(), MFE = MF.end();
401 3753228 : MFI!=MFE; ++MFI) {
402 2091781 : visitMachineBasicBlockBefore(&*MFI);
403 : // Keep track of the current bundle header.
404 : const MachineInstr *CurBundle = nullptr;
405 : // Do we expect the next instruction to be part of the same bundle?
406 : bool InBundle = false;
407 :
408 : for (MachineBasicBlock::const_instr_iterator MBBI = MFI->instr_begin(),
409 27023759 : MBBE = MFI->instr_end(); MBBI != MBBE; ++MBBI) {
410 24931978 : if (MBBI->getParent() != &*MFI) {
411 0 : report("Bad instruction parent pointer", &*MFI);
412 0 : errs() << "Instruction: " << *MBBI;
413 0 : continue;
414 : }
415 :
416 : // Check for consistent bundle flags.
417 24931978 : if (InBundle && !MBBI->isBundledWithPred())
418 0 : report("Missing BundledPred flag, "
419 : "BundledSucc was set on predecessor",
420 : &*MBBI);
421 24931978 : if (!InBundle && MBBI->isBundledWithPred())
422 0 : report("BundledPred flag is set, "
423 : "but BundledSucc not set on predecessor",
424 : &*MBBI);
425 :
426 : // Is this a bundle header?
427 24931978 : if (!MBBI->isInsideBundle()) {
428 24903027 : if (CurBundle)
429 22828094 : visitMachineBundleAfter(CurBundle);
430 : CurBundle = &*MBBI;
431 24903027 : visitMachineBundleBefore(CurBundle);
432 28951 : } else if (!CurBundle)
433 0 : report("No bundle header", &*MBBI);
434 24931978 : visitMachineInstrBefore(&*MBBI);
435 123768345 : for (unsigned I = 0, E = MBBI->getNumOperands(); I != E; ++I) {
436 : const MachineInstr &MI = *MBBI;
437 98836367 : const MachineOperand &Op = MI.getOperand(I);
438 98836367 : if (Op.getParent() != &MI) {
439 : // Make sure to use correct addOperand / RemoveOperand / ChangeTo
440 : // functions when replacing operands of a MachineInstr.
441 0 : report("Instruction has operand with wrong parent set", &MI);
442 : }
443 :
444 98836367 : visitMachineOperand(&Op, I);
445 : }
446 :
447 : visitMachineInstrAfter(&*MBBI);
448 :
449 : // Was this the last bundled instruction?
450 : InBundle = MBBI->isBundledWithSucc();
451 : }
452 2091781 : if (CurBundle)
453 2074933 : visitMachineBundleAfter(CurBundle);
454 2091781 : if (InBundle)
455 0 : report("BundledSucc flag set on last instruction in block", &MFI->back());
456 2091781 : visitMachineBasicBlockAfter(&*MFI);
457 : }
458 1661447 : visitMachineFunctionAfter();
459 :
460 : // Clean up.
461 : regsLive.clear();
462 : regsDefined.clear();
463 : regsDead.clear();
464 : regsKilled.clear();
465 : regMasks.clear();
466 1661447 : MBBInfoMap.clear();
467 :
468 1661447 : return foundErrors;
469 : }
470 :
471 71 : void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
472 : assert(MF);
473 71 : errs() << '\n';
474 71 : if (!foundErrors++) {
475 41 : if (Banner)
476 3 : errs() << "# " << Banner << '\n';
477 41 : if (LiveInts != nullptr)
478 2 : LiveInts->print(errs());
479 : else
480 39 : MF->print(errs(), Indexes);
481 : }
482 71 : errs() << "*** Bad machine code: " << msg << " ***\n"
483 71 : << "- function: " << MF->getName() << "\n";
484 71 : }
485 :
486 71 : void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
487 : assert(MBB);
488 71 : report(msg, MBB->getParent());
489 71 : errs() << "- basic block: " << printMBBReference(*MBB) << ' '
490 71 : << MBB->getName() << " (" << (const void *)MBB << ')';
491 71 : if (Indexes)
492 5 : errs() << " [" << Indexes->getMBBStartIdx(MBB)
493 5 : << ';' << Indexes->getMBBEndIdx(MBB) << ')';
494 71 : errs() << '\n';
495 71 : }
496 :
497 71 : void MachineVerifier::report(const char *msg, const MachineInstr *MI) {
498 : assert(MI);
499 71 : report(msg, MI->getParent());
500 71 : errs() << "- instruction: ";
501 71 : if (Indexes && Indexes->hasIndex(*MI))
502 10 : errs() << Indexes->getInstructionIndex(*MI) << '\t';
503 71 : MI->print(errs(), /*SkipOpers=*/true);
504 71 : }
505 :
506 17 : void MachineVerifier::report(const char *msg, const MachineOperand *MO,
507 : unsigned MONum, LLT MOVRegType) {
508 : assert(MO);
509 17 : report(msg, MO->getParent());
510 34 : errs() << "- operand " << MONum << ": ";
511 17 : MO->print(errs(), MOVRegType, TRI);
512 17 : errs() << "\n";
513 17 : }
514 :
515 0 : void MachineVerifier::report_context(SlotIndex Pos) const {
516 0 : errs() << "- at: " << Pos << '\n';
517 0 : }
518 :
519 0 : void MachineVerifier::report_context(const LiveInterval &LI) const {
520 0 : errs() << "- interval: " << LI << '\n';
521 0 : }
522 :
523 0 : void MachineVerifier::report_context(const LiveRange &LR, unsigned VRegUnit,
524 : LaneBitmask LaneMask) const {
525 0 : report_context_liverange(LR);
526 0 : report_context_vreg_regunit(VRegUnit);
527 0 : if (LaneMask.any())
528 0 : report_context_lanemask(LaneMask);
529 0 : }
530 :
531 0 : void MachineVerifier::report_context(const LiveRange::Segment &S) const {
532 0 : errs() << "- segment: " << S << '\n';
533 0 : }
534 :
535 0 : void MachineVerifier::report_context(const VNInfo &VNI) const {
536 0 : errs() << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n";
537 0 : }
538 :
539 0 : void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
540 0 : errs() << "- liverange: " << LR << '\n';
541 0 : }
542 :
543 0 : void MachineVerifier::report_context_vreg(unsigned VReg) const {
544 0 : errs() << "- v. register: " << printReg(VReg, TRI) << '\n';
545 0 : }
546 :
547 5 : void MachineVerifier::report_context_vreg_regunit(unsigned VRegOrUnit) const {
548 5 : if (TargetRegisterInfo::isVirtualRegister(VRegOrUnit)) {
549 5 : report_context_vreg(VRegOrUnit);
550 : } else {
551 0 : errs() << "- regunit: " << printRegUnit(VRegOrUnit, TRI) << '\n';
552 : }
553 5 : }
554 :
555 0 : void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {
556 0 : errs() << "- lanemask: " << PrintLaneMask(LaneMask) << '\n';
557 0 : }
558 :
559 2274209 : void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
560 2274209 : BBInfo &MInfo = MBBInfoMap[MBB];
561 2274209 : if (!MInfo.reachable) {
562 2090507 : MInfo.reachable = true;
563 2090507 : for (MachineBasicBlock::const_succ_iterator SuI = MBB->succ_begin(),
564 2703408 : SuE = MBB->succ_end(); SuI != SuE; ++SuI)
565 612901 : markReachable(*SuI);
566 : }
567 2274209 : }
568 :
569 1661447 : void MachineVerifier::visitMachineFunctionBefore() {
570 1661447 : lastIndex = SlotIndex();
571 3322894 : regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()
572 1661447 : : TRI->getReservedRegs(*MF);
573 :
574 3322894 : if (!MF->empty())
575 1661308 : markReachable(&MF->front());
576 :
577 : // Build a set of the basic blocks in the function.
578 1661447 : FunctionBlocks.clear();
579 3753228 : for (const auto &MBB : *MF) {
580 2091781 : FunctionBlocks.insert(&MBB);
581 2091781 : BBInfo &MInfo = MBBInfoMap[&MBB];
582 :
583 2091781 : MInfo.Preds.insert(MBB.pred_begin(), MBB.pred_end());
584 4183562 : if (MInfo.Preds.size() != MBB.pred_size())
585 0 : report("MBB has duplicate entries in its predecessor list.", &MBB);
586 :
587 2091781 : MInfo.Succs.insert(MBB.succ_begin(), MBB.succ_end());
588 4183562 : if (MInfo.Succs.size() != MBB.succ_size())
589 0 : report("MBB has duplicate entries in its successor list.", &MBB);
590 : }
591 :
592 : // Check that the register use lists are sane.
593 1661447 : MRI->verifyUseLists();
594 :
595 3322894 : if (!MF->empty())
596 1661308 : verifyStackFrame();
597 1661447 : }
598 :
599 : // Does iterator point to a and b as the first two elements?
600 : static bool matchPair(MachineBasicBlock::const_succ_iterator i,
601 : const MachineBasicBlock *a, const MachineBasicBlock *b) {
602 173728 : if (*i == a)
603 88753 : return *++i == b;
604 84975 : if (*i == b)
605 84975 : return *++i == a;
606 : return false;
607 : }
608 :
609 : void
610 2091781 : MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
611 2091781 : FirstTerminator = nullptr;
612 2091781 : FirstNonPHI = nullptr;
613 :
614 2091781 : if (!MF->getProperties().hasProperty(
615 2091781 : MachineFunctionProperties::Property::NoPHIs) && MRI->tracksLiveness()) {
616 : // If this block has allocatable physical registers live-in, check that
617 : // it is an entry block or landing pad.
618 1899566 : for (const auto &LI : MBB->liveins()) {
619 1132468 : if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
620 2258794 : MBB->getIterator() != MBB->getParent()->begin()) {
621 0 : report("MBB has allocatable live-in, but isn't entry or landing-pad.", MBB);
622 : }
623 : }
624 : }
625 :
626 : // Count the number of landing pad successors.
627 : SmallPtrSet<MachineBasicBlock*, 4> LandingPadSuccs;
628 : for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
629 2704992 : E = MBB->succ_end(); I != E; ++I) {
630 613211 : if ((*I)->isEHPad())
631 7983 : LandingPadSuccs.insert(*I);
632 613211 : if (!FunctionBlocks.count(*I))
633 0 : report("MBB has successor that isn't part of the function.", MBB);
634 613211 : if (!MBBInfoMap[*I].Preds.count(MBB)) {
635 0 : report("Inconsistent CFG", MBB);
636 0 : errs() << "MBB is not in the predecessor list of the successor "
637 0 : << printMBBReference(*(*I)) << ".\n";
638 : }
639 : }
640 :
641 : // Check the predecessor list.
642 : for (MachineBasicBlock::const_pred_iterator I = MBB->pred_begin(),
643 2704992 : E = MBB->pred_end(); I != E; ++I) {
644 613211 : if (!FunctionBlocks.count(*I))
645 0 : report("MBB has predecessor that isn't part of the function.", MBB);
646 613211 : if (!MBBInfoMap[*I].Succs.count(MBB)) {
647 0 : report("Inconsistent CFG", MBB);
648 0 : errs() << "MBB is not in the successor list of the predecessor "
649 0 : << printMBBReference(*(*I)) << ".\n";
650 : }
651 : }
652 :
653 2091781 : const MCAsmInfo *AsmInfo = TM->getMCAsmInfo();
654 2091781 : const BasicBlock *BB = MBB->getBasicBlock();
655 2091781 : const Function &F = MF->getFunction();
656 2091781 : if (LandingPadSuccs.size() > 1 &&
657 0 : !(AsmInfo &&
658 278 : AsmInfo->getExceptionHandlingType() == ExceptionHandling::SjLj &&
659 2091781 : BB && isa<SwitchInst>(BB->getTerminator())) &&
660 278 : !isScopedEHPersonality(classifyEHPersonality(F.getPersonalityFn())))
661 0 : report("MBB has more than one landing pad successor", MBB);
662 :
663 : // Call AnalyzeBranch. If it succeeds, there several more conditions to check.
664 2091781 : MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
665 : SmallVector<MachineOperand, 4> Cond;
666 2091781 : if (!TII->analyzeBranch(*const_cast<MachineBasicBlock *>(MBB), TBB, FBB,
667 2091781 : Cond)) {
668 : // Ok, AnalyzeBranch thinks it knows what's going on with this block. Let's
669 : // check whether its answers match up with reality.
670 400839 : if (!TBB && !FBB) {
671 : // Block falls through to its successor.
672 176829 : MachineFunction::const_iterator MBBI = MBB->getIterator();
673 : ++MBBI;
674 353658 : if (MBBI == MF->end()) {
675 : // It's possible that the block legitimately ends with a noreturn
676 : // call or an unreachable, in which case it won't actually fall
677 : // out the bottom of the function.
678 287156 : } else if (MBB->succ_size() == LandingPadSuccs.size()) {
679 : // It's possible that the block legitimately ends with a noreturn
680 : // call or an unreachable, in which case it won't actuall fall
681 : // out of the block.
682 133910 : } else if (MBB->succ_size() != 1+LandingPadSuccs.size()) {
683 0 : report("MBB exits via unconditional fall-through but doesn't have "
684 : "exactly one CFG successor!", MBB);
685 133910 : } else if (!MBB->isSuccessor(&*MBBI)) {
686 0 : report("MBB exits via unconditional fall-through but its successor "
687 : "differs from its CFG successor!", MBB);
688 : }
689 336828 : if (!MBB->empty() && MBB->back().isBarrier() &&
690 18 : !TII->isPredicated(MBB->back())) {
691 0 : report("MBB exits via unconditional fall-through but ends with a "
692 : "barrier instruction!", MBB);
693 : }
694 176829 : if (!Cond.empty()) {
695 0 : report("MBB exits via unconditional fall-through but has a condition!",
696 : MBB);
697 : }
698 224010 : } else if (TBB && !FBB && Cond.empty()) {
699 : // Block unconditionally branches somewhere.
700 : // If the block has exactly one successor, that happens to be a
701 : // landingpad, accept it as valid control flow.
702 100154 : if (MBB->succ_size() != 1+LandingPadSuccs.size() &&
703 0 : (MBB->succ_size() != 1 || LandingPadSuccs.size() != 1 ||
704 0 : *MBB->succ_begin() != *LandingPadSuccs.begin())) {
705 0 : report("MBB exits via unconditional branch but doesn't have "
706 : "exactly one CFG successor!", MBB);
707 50077 : } else if (!MBB->isSuccessor(TBB)) {
708 0 : report("MBB exits via unconditional branch but the CFG "
709 : "successor doesn't match the actual successor!", MBB);
710 : }
711 50077 : if (MBB->empty()) {
712 0 : report("MBB exits via unconditional branch but doesn't contain "
713 : "any instructions!", MBB);
714 50077 : } else if (!MBB->back().isBarrier()) {
715 0 : report("MBB exits via unconditional branch but doesn't end with a "
716 : "barrier instruction!", MBB);
717 50077 : } else if (!MBB->back().isTerminator()) {
718 0 : report("MBB exits via unconditional branch but the branch isn't a "
719 : "terminator instruction!", MBB);
720 : }
721 173933 : } else if (TBB && !FBB && !Cond.empty()) {
722 : // Block conditionally branches somewhere, otherwise falls through.
723 79712 : MachineFunction::const_iterator MBBI = MBB->getIterator();
724 : ++MBBI;
725 159424 : if (MBBI == MF->end()) {
726 0 : report("MBB conditionally falls through out of function!", MBB);
727 79712 : } else if (MBB->succ_size() == 1) {
728 : // A conditional branch with only one successor is weird, but allowed.
729 49 : if (&*MBBI != TBB)
730 0 : report("MBB exits via conditional branch/fall-through but only has "
731 : "one CFG successor!", MBB);
732 49 : else if (TBB != *MBB->succ_begin())
733 0 : report("MBB exits via conditional branch/fall-through but the CFG "
734 : "successor don't match the actual successor!", MBB);
735 79663 : } else if (MBB->succ_size() != 2) {
736 0 : report("MBB exits via conditional branch/fall-through but doesn't have "
737 : "exactly two CFG successors!", MBB);
738 79663 : } else if (!matchPair(MBB->succ_begin(), TBB, &*MBBI)) {
739 0 : report("MBB exits via conditional branch/fall-through but the CFG "
740 : "successors don't match the actual successors!", MBB);
741 : }
742 79712 : if (MBB->empty()) {
743 0 : report("MBB exits via conditional branch/fall-through but doesn't "
744 : "contain any instructions!", MBB);
745 79712 : } else if (MBB->back().isBarrier()) {
746 0 : report("MBB exits via conditional branch/fall-through but ends with a "
747 : "barrier instruction!", MBB);
748 79712 : } else if (!MBB->back().isTerminator()) {
749 0 : report("MBB exits via conditional branch/fall-through but the branch "
750 : "isn't a terminator instruction!", MBB);
751 : }
752 94221 : } else if (TBB && FBB) {
753 : // Block conditionally branches somewhere, otherwise branches
754 : // somewhere else.
755 94221 : if (MBB->succ_size() == 1) {
756 : // A conditional branch with only one successor is weird, but allowed.
757 156 : if (FBB != TBB)
758 0 : report("MBB exits via conditional branch/branch through but only has "
759 : "one CFG successor!", MBB);
760 156 : else if (TBB != *MBB->succ_begin())
761 0 : report("MBB exits via conditional branch/branch through but the CFG "
762 : "successor don't match the actual successor!", MBB);
763 94065 : } else if (MBB->succ_size() != 2) {
764 0 : report("MBB exits via conditional branch/branch but doesn't have "
765 : "exactly two CFG successors!", MBB);
766 94065 : } else if (!matchPair(MBB->succ_begin(), TBB, FBB)) {
767 0 : report("MBB exits via conditional branch/branch but the CFG "
768 : "successors don't match the actual successors!", MBB);
769 : }
770 94221 : if (MBB->empty()) {
771 0 : report("MBB exits via conditional branch/branch but doesn't "
772 : "contain any instructions!", MBB);
773 94221 : } else if (!MBB->back().isBarrier()) {
774 0 : report("MBB exits via conditional branch/branch but doesn't end with a "
775 : "barrier instruction!", MBB);
776 94221 : } else if (!MBB->back().isTerminator()) {
777 0 : report("MBB exits via conditional branch/branch but the branch "
778 : "isn't a terminator instruction!", MBB);
779 : }
780 94221 : if (Cond.empty()) {
781 0 : report("MBB exits via conditinal branch/branch but there's no "
782 : "condition!", MBB);
783 : }
784 : } else {
785 0 : report("AnalyzeBranch returned invalid data!", MBB);
786 : }
787 : }
788 :
789 : regsLive.clear();
790 4183562 : if (MRI->tracksLiveness()) {
791 5832232 : for (const auto &LI : MBB->liveins()) {
792 3772122 : if (!TargetRegisterInfo::isPhysicalRegister(LI.PhysReg)) {
793 0 : report("MBB live-in list contains non-physical register", MBB);
794 0 : continue;
795 : }
796 3772122 : for (MCSubRegIterator SubRegs(LI.PhysReg, TRI, /*IncludeSelf=*/true);
797 11883449 : SubRegs.isValid(); ++SubRegs)
798 8111327 : regsLive.insert(*SubRegs);
799 : }
800 : }
801 :
802 2091781 : const MachineFrameInfo &MFI = MF->getFrameInfo();
803 2091781 : BitVector PR = MFI.getPristineRegs(*MF);
804 18803254 : for (unsigned I : PR.set_bits()) {
805 16711473 : for (MCSubRegIterator SubRegs(I, TRI, /*IncludeSelf=*/true);
806 44787680 : SubRegs.isValid(); ++SubRegs)
807 28076207 : regsLive.insert(*SubRegs);
808 : }
809 :
810 : regsKilled.clear();
811 : regsDefined.clear();
812 :
813 2091781 : if (Indexes)
814 295334 : lastIndex = Indexes->getMBBStartIdx(MBB);
815 2091781 : }
816 :
817 : // This function gets called for all bundle headers, including normal
818 : // stand-alone unbundled instructions.
819 24903027 : void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
820 24903027 : if (Indexes && Indexes->hasIndex(*MI)) {
821 3283197 : SlotIndex idx = Indexes->getInstructionIndex(*MI);
822 3283197 : if (!(idx > lastIndex)) {
823 0 : report("Instruction index out of order", MI);
824 0 : errs() << "Last instruction was at " << lastIndex << '\n';
825 : }
826 3283197 : lastIndex = idx;
827 : }
828 :
829 : // Ensure non-terminators don't follow terminators.
830 : // Ignore predicated terminators formed by if conversion.
831 : // FIXME: If conversion shouldn't need to violate this rule.
832 24903027 : if (MI->isTerminator() && !TII->isPredicated(*MI)) {
833 2042614 : if (!FirstTerminator)
834 1925617 : FirstTerminator = MI;
835 22860413 : } else if (FirstTerminator) {
836 0 : report("Non-terminator instruction after the first terminator", MI);
837 0 : errs() << "First terminator was:\t" << *FirstTerminator;
838 : }
839 24903027 : }
840 :
841 : // The operands on an INLINEASM instruction must follow a template.
842 : // Verify that the flag operands make sense.
843 122512 : void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
844 : // The first two operands on INLINEASM are the asm string and global flags.
845 122512 : if (MI->getNumOperands() < 2) {
846 0 : report("Too few operands on inline asm", MI);
847 0 : return;
848 : }
849 245024 : if (!MI->getOperand(0).isSymbol())
850 0 : report("Asm string must be an external symbol", MI);
851 245024 : if (!MI->getOperand(1).isImm())
852 0 : report("Asm flags must be an immediate", MI);
853 : // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
854 : // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,
855 : // and Extra_IsConvergent = 32.
856 122512 : if (!isUInt<6>(MI->getOperand(1).getImm()))
857 0 : report("Unknown asm flags", &MI->getOperand(1), 1);
858 :
859 : static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");
860 :
861 : unsigned OpNo = InlineAsm::MIOp_FirstOperand;
862 : unsigned NumOps;
863 585868 : for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
864 464149 : const MachineOperand &MO = MI->getOperand(OpNo);
865 : // There may be implicit ops after the fixed operands.
866 464149 : if (!MO.isImm())
867 : break;
868 926712 : NumOps = 1 + InlineAsm::getNumOperandRegisters(MO.getImm());
869 : }
870 :
871 122512 : if (OpNo > MI->getNumOperands())
872 0 : report("Missing operands in last group", MI);
873 :
874 : // An optional MDNode follows the groups.
875 122512 : if (OpNo < MI->getNumOperands() && MI->getOperand(OpNo).isMetadata())
876 434 : ++OpNo;
877 :
878 : // All trailing operands must be implicit registers.
879 123067 : for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
880 555 : const MachineOperand &MO = MI->getOperand(OpNo);
881 555 : if (!MO.isReg() || !MO.isImplicit())
882 0 : report("Expected implicit register after groups", &MO, OpNo);
883 : }
884 : }
885 :
886 24931978 : void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
887 24931978 : const MCInstrDesc &MCID = MI->getDesc();
888 49863956 : if (MI->getNumOperands() < MCID.getNumOperands()) {
889 10 : report("Too few operands", MI);
890 10 : errs() << MCID.getNumOperands() << " operands expected, but "
891 10 : << MI->getNumOperands() << " given.\n";
892 : }
893 :
894 : if (MI->isPHI()) {
895 166362 : if (MF->getProperties().hasProperty(
896 : MachineFunctionProperties::Property::NoPHIs))
897 0 : report("Found PHI instruction with NoPHIs property set", MI);
898 :
899 83181 : if (FirstNonPHI)
900 1 : report("Found PHI instruction after non-PHI", MI);
901 24848797 : } else if (FirstNonPHI == nullptr)
902 2074784 : FirstNonPHI = MI;
903 :
904 : // Check the tied operands.
905 24931978 : if (MI->isInlineAsm())
906 122512 : verifyInlineAsm(MI);
907 :
908 : // Check the MachineMemOperands for basic consistency.
909 4896706 : for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
910 : E = MI->memoperands_end();
911 29828684 : I != E; ++I) {
912 9793412 : if ((*I)->isLoad() && !MI->mayLoad())
913 0 : report("Missing mayLoad flag", MI);
914 9793412 : if ((*I)->isStore() && !MI->mayStore())
915 0 : report("Missing mayStore flag", MI);
916 : }
917 :
918 : // Debug values must not have a slot index.
919 : // Other instructions must have one, unless they are inside a bundle.
920 24931978 : if (LiveInts) {
921 2845254 : bool mapped = !LiveInts->isNotInMIMap(*MI);
922 : if (MI->isDebugInstr()) {
923 129 : if (mapped)
924 0 : report("Debug instruction has a slot index", MI);
925 2845125 : } else if (MI->isInsideBundle()) {
926 223 : if (mapped)
927 0 : report("Instruction inside bundle has a slot index", MI);
928 : } else {
929 2844902 : if (!mapped)
930 0 : report("Missing slot index", MI);
931 : }
932 : }
933 :
934 49863956 : if (isPreISelGenericOpcode(MCID.getOpcode())) {
935 19370 : if (isFunctionSelected)
936 1 : report("Unexpected generic instruction in a Selected function", MI);
937 :
938 : // Check types.
939 : SmallVector<LLT, 4> Types;
940 84095 : for (unsigned I = 0; I < MCID.getNumOperands(); ++I) {
941 90710 : if (!MCID.OpInfo[I].isGenericType())
942 : continue;
943 : // Generic instructions specify type equality constraints between some of
944 : // their operands. Make sure these are consistent.
945 39270 : size_t TypeIdx = MCID.OpInfo[I].getGenericTypeIndex();
946 78876 : Types.resize(std::max(TypeIdx + 1, Types.size()));
947 :
948 39270 : const MachineOperand *MO = &MI->getOperand(I);
949 39270 : LLT OpTy = MRI->getType(MO->getReg());
950 : // Don't report a type mismatch if there is no actual mismatch, only a
951 : // type missing, to reduce noise:
952 39270 : if (OpTy.isValid()) {
953 : // Only the first valid type for a type index will be printed: don't
954 : // overwrite it later so it's always clear which type was expected:
955 39265 : if (!Types[TypeIdx].isValid())
956 29921 : Types[TypeIdx] = OpTy;
957 9343 : else if (Types[TypeIdx] != OpTy)
958 1 : report("Type mismatch in generic instruction", MO, I, OpTy);
959 : } else {
960 : // Generic instructions must have types attached to their operands.
961 5 : report("Generic instruction is missing a virtual register type", MO, I);
962 : }
963 : }
964 :
965 : // Generic opcodes must not have physical register operands.
966 66824 : for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
967 47454 : const MachineOperand *MO = &MI->getOperand(I);
968 47454 : if (MO->isReg() && TargetRegisterInfo::isPhysicalRegister(MO->getReg()))
969 0 : report("Generic instruction cannot have physical register", MO, I);
970 : }
971 : }
972 :
973 24931978 : StringRef ErrorInfo;
974 24931978 : if (!TII->verifyInstruction(*MI, ErrorInfo))
975 25 : report(ErrorInfo.data(), MI);
976 :
977 : // Verify properties of various specific instruction types
978 49863956 : switch(MI->getOpcode()) {
979 : default:
980 : break;
981 3116 : case TargetOpcode::G_LOAD:
982 : case TargetOpcode::G_STORE:
983 : // Generic loads and stores must have a single MachineMemOperand
984 : // describing that access.
985 3116 : if (!MI->hasOneMemOperand())
986 0 : report("Generic instruction accessing memory must have one mem operand",
987 : MI);
988 : break;
989 127 : case TargetOpcode::G_PHI: {
990 254 : LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
991 254 : if (!DstTy.isValid() ||
992 254 : !std::all_of(MI->operands_begin() + 1, MI->operands_end(),
993 : [this, &DstTy](const MachineOperand &MO) {
994 : if (!MO.isReg())
995 : return true;
996 : LLT Ty = MRI->getType(MO.getReg());
997 : if (!Ty.isValid() || (Ty != DstTy))
998 : return false;
999 : return true;
1000 : }))
1001 1 : report("Generic Instruction G_PHI has operands with incompatible/missing "
1002 : "types",
1003 : MI);
1004 : break;
1005 : }
1006 3854 : case TargetOpcode::G_SEXT:
1007 : case TargetOpcode::G_ZEXT:
1008 : case TargetOpcode::G_ANYEXT:
1009 : case TargetOpcode::G_TRUNC:
1010 : case TargetOpcode::G_FPEXT:
1011 : case TargetOpcode::G_FPTRUNC: {
1012 : // Number of operands and presense of types is already checked (and
1013 : // reported in case of any issues), so no need to report them again. As
1014 : // we're trying to report as many issues as possible at once, however, the
1015 : // instructions aren't guaranteed to have the right number of operands or
1016 : // types attached to them at this point
1017 : assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
1018 7708 : if (MI->getNumOperands() < MCID.getNumOperands())
1019 : break;
1020 3854 : LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
1021 7708 : LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
1022 3854 : if (!DstTy.isValid() || !SrcTy.isValid())
1023 : break;
1024 :
1025 3854 : LLT DstElTy = DstTy.isVector() ? DstTy.getElementType() : DstTy;
1026 3854 : LLT SrcElTy = SrcTy.isVector() ? SrcTy.getElementType() : SrcTy;
1027 7706 : if (DstElTy.isPointer() || SrcElTy.isPointer())
1028 3 : report("Generic extend/truncate can not operate on pointers", MI);
1029 :
1030 7708 : if (DstTy.isVector() != SrcTy.isVector()) {
1031 2 : report("Generic extend/truncate must be all-vector or all-scalar", MI);
1032 : // Generally we try to report as many issues as possible at once, but in
1033 : // this case it's not clear what should we be comparing the size of the
1034 : // scalar with: the size of the whole vector or its lane. Instead of
1035 : // making an arbitrary choice and emitting not so helpful message, let's
1036 : // avoid the extra noise and stop here.
1037 2 : break;
1038 : }
1039 3974 : if (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements())
1040 2 : report("Generic vector extend/truncate must preserve number of lanes",
1041 : MI);
1042 3852 : unsigned DstSize = DstElTy.getSizeInBits();
1043 3852 : unsigned SrcSize = SrcElTy.getSizeInBits();
1044 3852 : switch (MI->getOpcode()) {
1045 1663 : default:
1046 1663 : if (DstSize <= SrcSize)
1047 2 : report("Generic extend has destination type no larger than source", MI);
1048 : break;
1049 2189 : case TargetOpcode::G_TRUNC:
1050 : case TargetOpcode::G_FPTRUNC:
1051 2189 : if (DstSize >= SrcSize)
1052 1 : report("Generic truncate has destination type no smaller than source",
1053 : MI);
1054 : break;
1055 : }
1056 : break;
1057 : }
1058 5820933 : case TargetOpcode::COPY: {
1059 5820933 : if (foundErrors)
1060 : break;
1061 5820896 : const MachineOperand &DstOp = MI->getOperand(0);
1062 : const MachineOperand &SrcOp = MI->getOperand(1);
1063 5820896 : LLT DstTy = MRI->getType(DstOp.getReg());
1064 9292469 : LLT SrcTy = MRI->getType(SrcOp.getReg());
1065 5820896 : if (SrcTy.isValid() && DstTy.isValid()) {
1066 : // If both types are valid, check that the types are the same.
1067 779 : if (SrcTy != DstTy) {
1068 1 : report("Copy Instruction is illegal with mismatching types", MI);
1069 3 : errs() << "Def = " << DstTy << ", Src = " << SrcTy << "\n";
1070 : }
1071 : }
1072 5820896 : if (SrcTy.isValid() || DstTy.isValid()) {
1073 : // If one of them have valid types, let's just check they have the same
1074 : // size.
1075 17473 : unsigned SrcSize = TRI->getRegSizeInBits(SrcOp.getReg(), *MRI);
1076 17473 : unsigned DstSize = TRI->getRegSizeInBits(DstOp.getReg(), *MRI);
1077 : assert(SrcSize && "Expecting size here");
1078 : assert(DstSize && "Expecting size here");
1079 17473 : if (SrcSize != DstSize)
1080 2 : if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {
1081 2 : report("Copy Instruction is illegal with mismatching sizes", MI);
1082 4 : errs() << "Def Size = " << DstSize << ", Src Size = " << SrcSize
1083 2 : << "\n";
1084 : }
1085 : }
1086 : break;
1087 : }
1088 3045 : case TargetOpcode::STATEPOINT:
1089 3045 : if (!MI->getOperand(StatepointOpers::IDPos).isImm() ||
1090 6090 : !MI->getOperand(StatepointOpers::NBytesPos).isImm() ||
1091 : !MI->getOperand(StatepointOpers::NCallArgsPos).isImm())
1092 0 : report("meta operands to STATEPOINT not constant!", MI);
1093 : break;
1094 :
1095 : auto VerifyStackMapConstant = [&](unsigned Offset) {
1096 : if (!MI->getOperand(Offset).isImm() ||
1097 : MI->getOperand(Offset).getImm() != StackMaps::ConstantOp ||
1098 : !MI->getOperand(Offset + 1).isImm())
1099 : report("stack map constant to STATEPOINT not well formed!", MI);
1100 : };
1101 : const unsigned VarStart = StatepointOpers(MI).getVarIdx();
1102 : VerifyStackMapConstant(VarStart + StatepointOpers::CCOffset);
1103 : VerifyStackMapConstant(VarStart + StatepointOpers::FlagsOffset);
1104 : VerifyStackMapConstant(VarStart + StatepointOpers::NumDeoptOperandsOffset);
1105 :
1106 : // TODO: verify we have properly encoded deopt arguments
1107 : };
1108 24931978 : }
1109 :
1110 : void
1111 98836367 : MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
1112 98836367 : const MachineInstr *MI = MO->getParent();
1113 98836367 : const MCInstrDesc &MCID = MI->getDesc();
1114 98836367 : unsigned NumDefs = MCID.getNumDefs();
1115 98836367 : if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)
1116 35 : NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;
1117 :
1118 : // The first MCID.NumDefs operands must be explicit register defines
1119 98836367 : if (MONum < NumDefs) {
1120 20072778 : const MCOperandInfo &MCOI = MCID.OpInfo[MONum];
1121 20072778 : if (!MO->isReg())
1122 0 : report("Explicit definition must be a register", MO, MONum);
1123 20072778 : else if (!MO->isDef() && !MCOI.isOptionalDef())
1124 0 : report("Explicit definition marked as use", MO, MONum);
1125 20072778 : else if (MO->isImplicit())
1126 0 : report("Explicit definition marked as implicit", MO, MONum);
1127 157527178 : } else if (MONum < MCID.getNumOperands()) {
1128 58771416 : const MCOperandInfo &MCOI = MCID.OpInfo[MONum];
1129 : // Don't check if it's the last operand in a variadic instruction. See,
1130 : // e.g., LDM_RET in the arm back end.
1131 87277269 : if (MO->isReg() &&
1132 2040714 : !(MI->isVariadic() && MONum == MCID.getNumOperands()-1)) {
1133 27551969 : if (MO->isDef() && !MCOI.isOptionalDef())
1134 0 : report("Explicit operand marked as def", MO, MONum);
1135 27551969 : if (MO->isImplicit())
1136 0 : report("Explicit operand marked as implicit", MO, MONum);
1137 : }
1138 :
1139 58771416 : int TiedTo = MCID.getOperandConstraint(MONum, MCOI::TIED_TO);
1140 : if (TiedTo != -1) {
1141 690324 : if (!MO->isReg())
1142 0 : report("Tied use must be a register", MO, MONum);
1143 690324 : else if (!MO->isTied())
1144 0 : report("Operand should be tied", MO, MONum);
1145 690324 : else if (unsigned(TiedTo) != MI->findTiedOperandIdx(MONum))
1146 0 : report("Tied def doesn't match MCInstrDesc", MO, MONum);
1147 1380648 : else if (TargetRegisterInfo::isPhysicalRegister(MO->getReg())) {
1148 397217 : const MachineOperand &MOTied = MI->getOperand(TiedTo);
1149 397217 : if (!MOTied.isReg())
1150 0 : report("Tied counterpart must be a register", &MOTied, TiedTo);
1151 794434 : else if (TargetRegisterInfo::isPhysicalRegister(MOTied.getReg()) &&
1152 : MO->getReg() != MOTied.getReg())
1153 1 : report("Tied physical registers must match.", &MOTied, TiedTo);
1154 : }
1155 58081092 : } else if (MO->isReg() && MO->isTied())
1156 0 : report("Explicit operand should not be tied", MO, MONum);
1157 : } else {
1158 : // ARM adds %reg0 operands to indicate predicates. We'll allow that.
1159 22306659 : if (MO->isReg() && !MO->isImplicit() && !MI->isVariadic() && MO->getReg())
1160 0 : report("Extra explicit operand on non-variadic instruction", MO, MONum);
1161 : }
1162 :
1163 98836367 : switch (MO->getType()) {
1164 64835283 : case MachineOperand::MO_Register: {
1165 64835283 : const unsigned Reg = MO->getReg();
1166 64835283 : if (!Reg)
1167 1769825 : return;
1168 126130920 : if (MRI->tracksLiveness() && !MI->isDebugValue())
1169 62235260 : checkLiveness(MO, MONum);
1170 :
1171 : // Verify the consistency of tied operands.
1172 63065460 : if (MO->isTied()) {
1173 1394054 : unsigned OtherIdx = MI->findTiedOperandIdx(MONum);
1174 1394054 : const MachineOperand &OtherMO = MI->getOperand(OtherIdx);
1175 1394054 : if (!OtherMO.isReg())
1176 0 : report("Must be tied to a register", MO, MONum);
1177 1394054 : if (!OtherMO.isTied())
1178 0 : report("Missing tie flags on tied operand", MO, MONum);
1179 1394054 : if (MI->findTiedOperandIdx(OtherIdx) != MONum)
1180 0 : report("Inconsistent tie links", MO, MONum);
1181 2788108 : if (MONum < MCID.getNumDefs()) {
1182 1382620 : if (OtherIdx < MCID.getNumOperands()) {
1183 : if (-1 == MCID.getOperandConstraint(OtherIdx, MCOI::TIED_TO))
1184 0 : report("Explicit def tied to explicit use without tie constraint",
1185 : MO, MONum);
1186 : } else {
1187 986 : if (!OtherMO.isImplicit())
1188 0 : report("Explicit def should be tied to implicit use", MO, MONum);
1189 : }
1190 : }
1191 : }
1192 :
1193 : // Verify two-address constraints after leaving SSA form.
1194 : unsigned DefIdx;
1195 84997658 : if (!MRI->isSSA() && MO->isUse() &&
1196 84997658 : MI->isRegTiedToDefOperand(MONum, &DefIdx) &&
1197 919202 : Reg != MI->getOperand(DefIdx).getReg())
1198 0 : report("Two-address instruction operands must be identical", MO, MONum);
1199 :
1200 : // Check register classes.
1201 : unsigned SubIdx = MO->getSubReg();
1202 :
1203 63065460 : if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
1204 36874096 : if (SubIdx) {
1205 0 : report("Illegal subregister index for physical register", MO, MONum);
1206 0 : return;
1207 : }
1208 73748192 : if (MONum < MCID.getNumOperands()) {
1209 22485778 : if (const TargetRegisterClass *DRC =
1210 22485778 : TII->getRegClass(MCID, MONum, TRI, *MF)) {
1211 18572159 : if (!DRC->contains(Reg)) {
1212 0 : report("Illegal physical register for instruction", MO, MONum);
1213 0 : errs() << printReg(Reg, TRI) << " is not a "
1214 0 : << TRI->getRegClassName(DRC) << " register.\n";
1215 : }
1216 : }
1217 : }
1218 36874096 : if (MO->isRenamable()) {
1219 24835958 : if (MRI->isReserved(Reg)) {
1220 0 : report("isRenamable set on reserved register", MO, MONum);
1221 0 : return;
1222 : }
1223 : }
1224 36874096 : if (MI->isDebugValue() && MO->isUse() && !MO->isDebug()) {
1225 0 : report("Use-reg is not IsDebug in a DBG_VALUE", MO, MONum);
1226 0 : return;
1227 : }
1228 : } else {
1229 : // Virtual register.
1230 26191364 : const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);
1231 26147102 : if (!RC) {
1232 : // This is a generic virtual register.
1233 :
1234 : // If we're post-Select, we can't have gvregs anymore.
1235 59938 : if (isFunctionSelected) {
1236 1 : report("Generic virtual register invalid in a Selected function",
1237 : MO, MONum);
1238 2 : return;
1239 : }
1240 :
1241 : // The gvreg must have a type and it must not have a SubIdx.
1242 59937 : LLT Ty = MRI->getType(Reg);
1243 59937 : if (!Ty.isValid()) {
1244 0 : report("Generic virtual register must have a valid type", MO,
1245 : MONum);
1246 0 : return;
1247 : }
1248 :
1249 : const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
1250 :
1251 : // If we're post-RegBankSelect, the gvreg must have a bank.
1252 59937 : if (!RegBank && isFunctionRegBankSelected) {
1253 1 : report("Generic virtual register must have a bank in a "
1254 : "RegBankSelected function",
1255 : MO, MONum);
1256 1 : return;
1257 : }
1258 :
1259 : // Make sure the register fits into its register bank if any.
1260 59936 : if (RegBank && Ty.isValid() &&
1261 24217 : RegBank->getSize() < Ty.getSizeInBits()) {
1262 0 : report("Register bank is too small for virtual register", MO,
1263 : MONum);
1264 0 : errs() << "Register bank " << RegBank->getName() << " too small("
1265 0 : << RegBank->getSize() << ") to fit " << Ty.getSizeInBits()
1266 0 : << "-bits\n";
1267 0 : return;
1268 : }
1269 59936 : if (SubIdx) {
1270 0 : report("Generic virtual register does not subregister index", MO,
1271 : MONum);
1272 0 : return;
1273 : }
1274 :
1275 : // If this is a target specific instruction and this operand
1276 : // has register class constraint, the virtual register must
1277 : // comply to it.
1278 59936 : if (!isPreISelGenericOpcode(MCID.getOpcode()) &&
1279 78412 : MONum < MCID.getNumOperands() &&
1280 18476 : TII->getRegClass(MCID, MONum, TRI, *MF)) {
1281 0 : report("Virtual register does not match instruction constraint", MO,
1282 : MONum);
1283 0 : errs() << "Expect register class "
1284 0 : << TRI->getRegClassName(
1285 0 : TII->getRegClass(MCID, MONum, TRI, *MF))
1286 0 : << " but got nothing\n";
1287 0 : return;
1288 : }
1289 :
1290 59936 : break;
1291 : }
1292 26131426 : if (SubIdx) {
1293 : const TargetRegisterClass *SRC =
1294 3053628 : TRI->getSubClassWithSubReg(RC, SubIdx);
1295 3053628 : if (!SRC) {
1296 0 : report("Invalid subregister index for virtual register", MO, MONum);
1297 0 : errs() << "Register class " << TRI->getRegClassName(RC)
1298 0 : << " does not support subreg index " << SubIdx << "\n";
1299 0 : return;
1300 : }
1301 3053628 : if (RC != SRC) {
1302 0 : report("Invalid register class for subregister index", MO, MONum);
1303 0 : errs() << "Register class " << TRI->getRegClassName(RC)
1304 0 : << " does not fully support subreg index " << SubIdx << "\n";
1305 0 : return;
1306 : }
1307 : }
1308 52262852 : if (MONum < MCID.getNumOperands()) {
1309 24267903 : if (const TargetRegisterClass *DRC =
1310 24267903 : TII->getRegClass(MCID, MONum, TRI, *MF)) {
1311 14124661 : if (SubIdx) {
1312 : const TargetRegisterClass *SuperRC =
1313 1007863 : TRI->getLargestLegalSuperClass(RC, *MF);
1314 1007863 : if (!SuperRC) {
1315 0 : report("No largest legal super class exists.", MO, MONum);
1316 0 : return;
1317 : }
1318 1007863 : DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx);
1319 1007863 : if (!DRC) {
1320 0 : report("No matching super-reg register class.", MO, MONum);
1321 0 : return;
1322 : }
1323 : }
1324 14124661 : if (!RC->hasSuperClassEq(DRC)) {
1325 0 : report("Illegal virtual register for instruction", MO, MONum);
1326 0 : errs() << "Expected a " << TRI->getRegClassName(DRC)
1327 0 : << " register, but got a " << TRI->getRegClassName(RC)
1328 0 : << " register\n";
1329 : }
1330 : }
1331 : }
1332 : }
1333 : break;
1334 : }
1335 :
1336 222000 : case MachineOperand::MO_RegisterMask:
1337 222000 : regMasks.push_back(MO->getRegMask());
1338 222000 : break;
1339 :
1340 : case MachineOperand::MO_MachineBasicBlock:
1341 171528 : if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent()))
1342 0 : report("PHI operand is not in the CFG", MO, MONum);
1343 : break;
1344 :
1345 419377 : case MachineOperand::MO_FrameIndex:
1346 36598 : if (LiveStks && LiveStks->hasInterval(MO->getIndex()) &&
1347 426938 : LiveInts && !LiveInts->isNotInMIMap(*MI)) {
1348 4871 : int FI = MO->getIndex();
1349 4871 : LiveInterval &LI = LiveStks->getInterval(FI);
1350 4871 : SlotIndex Idx = LiveInts->getInstructionIndex(*MI);
1351 :
1352 4871 : bool stores = MI->mayStore();
1353 4871 : bool loads = MI->mayLoad();
1354 : // For a memory-to-memory move, we need to check if the frame
1355 : // index is used for storing or loading, by inspecting the
1356 : // memory operands.
1357 4871 : if (stores && loads) {
1358 44 : for (auto *MMO : MI->memoperands()) {
1359 : const PseudoSourceValue *PSV = MMO->getPseudoValue();
1360 26 : if (PSV == nullptr) continue;
1361 : const FixedStackPseudoSourceValue *Value =
1362 : dyn_cast<FixedStackPseudoSourceValue>(PSV);
1363 : if (Value == nullptr) continue;
1364 26 : if (Value->getFrameIndex() != FI) continue;
1365 :
1366 46 : if (MMO->isStore())
1367 : loads = false;
1368 : else
1369 : stores = false;
1370 : break;
1371 : }
1372 23 : if (loads == stores)
1373 0 : report("Missing fixed stack memoperand.", MI);
1374 : }
1375 4871 : if (loads && !LI.liveAt(Idx.getRegSlot(true))) {
1376 0 : report("Instruction loads from dead spill slot", MO, MONum);
1377 0 : errs() << "Live stack: " << LI << '\n';
1378 : }
1379 4871 : if (stores && !LI.liveAt(Idx.getRegSlot())) {
1380 0 : report("Instruction stores to dead spill slot", MO, MONum);
1381 0 : errs() << "Live stack: " << LI << '\n';
1382 : }
1383 : }
1384 : break;
1385 :
1386 : default:
1387 : break;
1388 : }
1389 : }
1390 :
1391 4963893 : void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
1392 : unsigned MONum, SlotIndex UseIdx, const LiveRange &LR, unsigned VRegOrUnit,
1393 : LaneBitmask LaneMask) {
1394 4963893 : LiveQueryResult LRQ = LR.Query(UseIdx);
1395 : // Check if we have a segment at the use, note however that we only need one
1396 : // live subregister range, the others may be dead.
1397 4963893 : if (!LRQ.valueIn() && LaneMask.none()) {
1398 0 : report("No live segment at use", MO, MONum);
1399 0 : report_context_liverange(LR);
1400 0 : report_context_vreg_regunit(VRegOrUnit);
1401 0 : report_context(UseIdx);
1402 : }
1403 4963893 : if (MO->isKill() && !LRQ.isKill()) {
1404 0 : report("Live range continues after kill flag", MO, MONum);
1405 0 : report_context_liverange(LR);
1406 0 : report_context_vreg_regunit(VRegOrUnit);
1407 0 : if (LaneMask.any())
1408 0 : report_context_lanemask(LaneMask);
1409 0 : report_context(UseIdx);
1410 : }
1411 4963893 : }
1412 :
1413 3186633 : void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
1414 : unsigned MONum, SlotIndex DefIdx, const LiveRange &LR, unsigned VRegOrUnit,
1415 : bool SubRangeCheck, LaneBitmask LaneMask) {
1416 3186633 : if (const VNInfo *VNI = LR.getVNInfoAt(DefIdx)) {
1417 : assert(VNI && "NULL valno is not allowed");
1418 3186633 : if (VNI->def != DefIdx) {
1419 0 : report("Inconsistent valno->def", MO, MONum);
1420 0 : report_context_liverange(LR);
1421 0 : report_context_vreg_regunit(VRegOrUnit);
1422 0 : if (LaneMask.any())
1423 0 : report_context_lanemask(LaneMask);
1424 0 : report_context(*VNI);
1425 0 : report_context(DefIdx);
1426 : }
1427 : } else {
1428 0 : report("No live segment at def", MO, MONum);
1429 0 : report_context_liverange(LR);
1430 0 : report_context_vreg_regunit(VRegOrUnit);
1431 0 : if (LaneMask.any())
1432 0 : report_context_lanemask(LaneMask);
1433 0 : report_context(DefIdx);
1434 : }
1435 : // Check that, if the dead def flag is present, LiveInts agree.
1436 3186633 : if (MO->isDead()) {
1437 40744 : LiveQueryResult LRQ = LR.Query(DefIdx);
1438 40744 : if (!LRQ.isDeadDef()) {
1439 : assert(TargetRegisterInfo::isVirtualRegister(VRegOrUnit) &&
1440 : "Expecting a virtual register.");
1441 : // A dead subreg def only tells us that the specific subreg is dead. There
1442 : // could be other non-dead defs of other subregs, or we could have other
1443 : // parts of the register being live through the instruction. So unless we
1444 : // are checking liveness for a subrange it is ok for the live range to
1445 : // continue, given that we have a dead def of a subregister.
1446 13 : if (SubRangeCheck || MO->getSubReg() == 0) {
1447 5 : report("Live range continues after dead def flag", MO, MONum);
1448 5 : report_context_liverange(LR);
1449 5 : report_context_vreg_regunit(VRegOrUnit);
1450 5 : if (LaneMask.any())
1451 3 : report_context_lanemask(LaneMask);
1452 : }
1453 : }
1454 : }
1455 3186633 : }
1456 :
1457 62235260 : void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
1458 62235260 : const MachineInstr *MI = MO->getParent();
1459 62235260 : const unsigned Reg = MO->getReg();
1460 :
1461 : // Both use and def operands can read a register.
1462 : if (MO->readsReg()) {
1463 39337987 : if (MO->isKill())
1464 9872822 : addRegWithSubRegs(regsKilled, Reg);
1465 :
1466 : // Check that LiveVars knows this kill.
1467 39337987 : if (LiveVars && TargetRegisterInfo::isVirtualRegister(Reg) &&
1468 : MO->isKill()) {
1469 516 : LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
1470 516 : if (!is_contained(VI.Kills, MI))
1471 0 : report("Kill missing from LiveVariables", MO, MONum);
1472 : }
1473 :
1474 : // Check LiveInts liveness and kill.
1475 39337987 : if (LiveInts && !LiveInts->isNotInMIMap(*MI)) {
1476 4769241 : SlotIndex UseIdx = LiveInts->getInstructionIndex(*MI);
1477 : // Check the cached regunit intervals.
1478 9538482 : if (TargetRegisterInfo::isPhysicalRegister(Reg) && !isReserved(Reg)) {
1479 1912480 : for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) {
1480 1480192 : if (MRI->isReservedRegUnit(*Units))
1481 : continue;
1482 1480124 : if (const LiveRange *LR = LiveInts->getCachedRegUnit(*Units))
1483 632562 : checkLivenessAtUse(MO, MONum, UseIdx, *LR, *Units);
1484 : }
1485 : }
1486 :
1487 9538482 : if (TargetRegisterInfo::isVirtualRegister(Reg)) {
1488 2894731 : if (LiveInts->hasInterval(Reg)) {
1489 : // This is a virtual register interval.
1490 2894731 : const LiveInterval &LI = LiveInts->getInterval(Reg);
1491 2894731 : checkLivenessAtUse(MO, MONum, UseIdx, LI, Reg);
1492 :
1493 2894731 : if (LI.hasSubRanges() && !MO->isDef()) {
1494 : unsigned SubRegIdx = MO->getSubReg();
1495 : LaneBitmask MOMask = SubRegIdx != 0
1496 509664 : ? TRI->getSubRegIndexLaneMask(SubRegIdx)
1497 798269 : : MRI->getMaxLaneMaskForVReg(Reg);
1498 : LaneBitmask LiveInMask;
1499 3830781 : for (const LiveInterval::SubRange &SR : LI.subranges()) {
1500 3032512 : if ((MOMask & SR.LaneMask).none())
1501 1595912 : continue;
1502 1436600 : checkLivenessAtUse(MO, MONum, UseIdx, SR, Reg, SR.LaneMask);
1503 1436600 : LiveQueryResult LRQ = SR.Query(UseIdx);
1504 1436600 : if (LRQ.valueIn())
1505 : LiveInMask |= SR.LaneMask;
1506 : }
1507 : // At least parts of the register has to be live at the use.
1508 798269 : if ((LiveInMask & MOMask).none()) {
1509 0 : report("No live subrange at use", MO, MONum);
1510 0 : report_context(LI);
1511 0 : report_context(UseIdx);
1512 : }
1513 : }
1514 : } else {
1515 0 : report("Virtual register has no live interval", MO, MONum);
1516 : }
1517 : }
1518 : }
1519 :
1520 : // Use of a dead register.
1521 : if (!regsLive.count(Reg)) {
1522 22942456 : if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
1523 : // Reserved registers may be used even when 'dead'.
1524 10962104 : bool Bad = !isReserved(Reg);
1525 : // We are fine if just any subregister has a defined value.
1526 10962104 : if (Bad) {
1527 1918537 : for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid();
1528 : ++SubRegs) {
1529 1990540 : if (regsLive.count(*SubRegs)) {
1530 : Bad = false;
1531 : break;
1532 : }
1533 : }
1534 : }
1535 : // If there is an additional implicit-use of a super register we stop
1536 : // here. By definition we are fine if the super register is not
1537 : // (completely) dead, if the complete super register is dead we will
1538 : // get a report for its operand.
1539 10962104 : if (Bad) {
1540 143 : for (const MachineOperand &MOP : MI->uses()) {
1541 91 : if (!MOP.isReg() || !MOP.isImplicit())
1542 : continue;
1543 :
1544 132 : if (!TargetRegisterInfo::isPhysicalRegister(MOP.getReg()))
1545 : continue;
1546 :
1547 274 : for (MCSubRegIterator SubRegs(MOP.getReg(), TRI); SubRegs.isValid();
1548 : ++SubRegs) {
1549 169 : if (*SubRegs == Reg) {
1550 : Bad = false;
1551 : break;
1552 : }
1553 : }
1554 : }
1555 : }
1556 10962104 : if (Bad)
1557 1 : report("Using an undefined physical register", MO, MONum);
1558 509124 : } else if (MRI->def_empty(Reg)) {
1559 0 : report("Reading virtual register without a def", MO, MONum);
1560 : } else {
1561 509124 : BBInfo &MInfo = MBBInfoMap[MI->getParent()];
1562 : // We don't know which virtual registers are live in, so only complain
1563 : // if vreg was killed in this MBB. Otherwise keep track of vregs that
1564 : // must be live in. PHI instructions are handled separately.
1565 : if (MInfo.regsKilled.count(Reg))
1566 0 : report("Using a killed virtual register", MO, MONum);
1567 509124 : else if (!MI->isPHI())
1568 337691 : MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI));
1569 : }
1570 : }
1571 : }
1572 :
1573 62235260 : if (MO->isDef()) {
1574 : // Register defined.
1575 : // TODO: verify that earlyclobber ops are not used.
1576 23121184 : if (MO->isDead())
1577 2103263 : addRegWithSubRegs(regsDead, Reg);
1578 : else
1579 21017921 : addRegWithSubRegs(regsDefined, Reg);
1580 :
1581 : // Verify SSA form.
1582 55601148 : if (MRI->isSSA() && TargetRegisterInfo::isVirtualRegister(Reg) &&
1583 9358780 : std::next(MRI->def_begin(Reg)) != MRI->def_end())
1584 0 : report("Multiple virtual register defs in SSA form", MO, MONum);
1585 :
1586 : // Check LiveInts for a live segment, but only for virtual registers.
1587 23121184 : if (LiveInts && !LiveInts->isNotInMIMap(*MI)) {
1588 2669364 : SlotIndex DefIdx = LiveInts->getInstructionIndex(*MI);
1589 : DefIdx = DefIdx.getRegSlot(MO->isEarlyClobber());
1590 :
1591 5338728 : if (TargetRegisterInfo::isVirtualRegister(Reg)) {
1592 2129925 : if (LiveInts->hasInterval(Reg)) {
1593 2129925 : const LiveInterval &LI = LiveInts->getInterval(Reg);
1594 2129925 : checkLivenessAtDef(MO, MONum, DefIdx, LI, Reg);
1595 :
1596 2129925 : if (LI.hasSubRanges()) {
1597 : unsigned SubRegIdx = MO->getSubReg();
1598 : LaneBitmask MOMask = SubRegIdx != 0
1599 642487 : ? TRI->getSubRegIndexLaneMask(SubRegIdx)
1600 757606 : : MRI->getMaxLaneMaskForVReg(Reg);
1601 3394395 : for (const LiveInterval::SubRange &SR : LI.subranges()) {
1602 5273578 : if ((SR.LaneMask & MOMask).none())
1603 : continue;
1604 1056708 : checkLivenessAtDef(MO, MONum, DefIdx, SR, Reg, true, SR.LaneMask);
1605 : }
1606 : }
1607 : } else {
1608 0 : report("Virtual register has no Live interval", MO, MONum);
1609 : }
1610 : }
1611 : }
1612 : }
1613 62235260 : }
1614 :
1615 0 : void MachineVerifier::visitMachineInstrAfter(const MachineInstr *MI) {}
1616 :
1617 : // This function gets called after visiting all instructions in a bundle. The
1618 : // argument points to the bundle header.
1619 : // Normal stand-alone instructions are also considered 'bundles', and this
1620 : // function is called for all of them.
1621 24903027 : void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {
1622 24903027 : BBInfo &MInfo = MBBInfoMap[MI->getParent()];
1623 24903027 : set_union(MInfo.regsKilled, regsKilled);
1624 : set_subtract(regsLive, regsKilled); regsKilled.clear();
1625 : // Kill any masked registers.
1626 25125027 : while (!regMasks.empty()) {
1627 : const uint32_t *Mask = regMasks.pop_back_val();
1628 5713868 : for (RegSet::iterator I = regsLive.begin(), E = regsLive.end(); I != E; ++I)
1629 10983736 : if (TargetRegisterInfo::isPhysicalRegister(*I) &&
1630 : MachineOperand::clobbersPhysReg(Mask, *I))
1631 2041720 : regsDead.push_back(*I);
1632 : }
1633 : set_subtract(regsLive, regsDead); regsDead.clear();
1634 24903027 : set_union(regsLive, regsDefined); regsDefined.clear();
1635 24903027 : }
1636 :
1637 : void
1638 2091781 : MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {
1639 2091781 : MBBInfoMap[MBB].regsLiveOut = regsLive;
1640 : regsLive.clear();
1641 :
1642 2091781 : if (Indexes) {
1643 295334 : SlotIndex stop = Indexes->getMBBEndIdx(MBB);
1644 295334 : if (!(stop > lastIndex)) {
1645 0 : report("Block ends before last instruction index", MBB);
1646 0 : errs() << "Block ends at " << stop
1647 0 : << " last instruction was at " << lastIndex << '\n';
1648 : }
1649 295334 : lastIndex = stop;
1650 : }
1651 2091781 : }
1652 :
1653 : // Calculate the largest possible vregsPassed sets. These are the registers that
1654 : // can pass through an MBB live, but may not be live every time. It is assumed
1655 : // that all vregsPassed sets are empty before the call.
1656 1661447 : void MachineVerifier::calcRegsPassed() {
1657 : // First push live-out regs to successors' vregsPassed. Remember the MBBs that
1658 : // have any vregsPassed.
1659 : SmallPtrSet<const MachineBasicBlock*, 8> todo;
1660 3753228 : for (const auto &MBB : *MF) {
1661 2091781 : BBInfo &MInfo = MBBInfoMap[&MBB];
1662 2091781 : if (!MInfo.reachable)
1663 : continue;
1664 : for (MachineBasicBlock::const_succ_iterator SuI = MBB.succ_begin(),
1665 2703408 : SuE = MBB.succ_end(); SuI != SuE; ++SuI) {
1666 612901 : BBInfo &SInfo = MBBInfoMap[*SuI];
1667 612901 : if (SInfo.addPassed(MInfo.regsLiveOut))
1668 210411 : todo.insert(*SuI);
1669 : }
1670 : }
1671 :
1672 : // Iteratively push vregsPassed to successors. This will converge to the same
1673 : // final state regardless of DenseSet iteration order.
1674 1909381 : while (!todo.empty()) {
1675 247934 : const MachineBasicBlock *MBB = *todo.begin();
1676 : todo.erase(MBB);
1677 247934 : BBInfo &MInfo = MBBInfoMap[MBB];
1678 247934 : for (MachineBasicBlock::const_succ_iterator SuI = MBB->succ_begin(),
1679 495751 : SuE = MBB->succ_end(); SuI != SuE; ++SuI) {
1680 247817 : if (*SuI == MBB)
1681 : continue;
1682 223897 : BBInfo &SInfo = MBBInfoMap[*SuI];
1683 223897 : if (SInfo.addPassed(MInfo.vregsPassed))
1684 148222 : todo.insert(*SuI);
1685 : }
1686 : }
1687 1661447 : }
1688 :
1689 : // Calculate the set of virtual registers that must be passed through each basic
1690 : // block in order to satisfy the requirements of successor blocks. This is very
1691 : // similar to calcRegsPassed, only backwards.
1692 1661447 : void MachineVerifier::calcRegsRequired() {
1693 : // First push live-in regs to predecessors' vregsRequired.
1694 : SmallPtrSet<const MachineBasicBlock*, 8> todo;
1695 3753228 : for (const auto &MBB : *MF) {
1696 2091781 : BBInfo &MInfo = MBBInfoMap[&MBB];
1697 : for (MachineBasicBlock::const_pred_iterator PrI = MBB.pred_begin(),
1698 2704992 : PrE = MBB.pred_end(); PrI != PrE; ++PrI) {
1699 613211 : BBInfo &PInfo = MBBInfoMap[*PrI];
1700 613211 : if (PInfo.addRequired(MInfo.vregsLiveIn))
1701 75117 : todo.insert(*PrI);
1702 : }
1703 : }
1704 :
1705 : // Iteratively push vregsRequired to predecessors. This will converge to the
1706 : // same final state regardless of DenseSet iteration order.
1707 1760442 : while (!todo.empty()) {
1708 98995 : const MachineBasicBlock *MBB = *todo.begin();
1709 : todo.erase(MBB);
1710 98995 : BBInfo &MInfo = MBBInfoMap[MBB];
1711 98995 : for (MachineBasicBlock::const_pred_iterator PrI = MBB->pred_begin(),
1712 239065 : PrE = MBB->pred_end(); PrI != PrE; ++PrI) {
1713 140070 : if (*PrI == MBB)
1714 : continue;
1715 121837 : BBInfo &SInfo = MBBInfoMap[*PrI];
1716 121837 : if (SInfo.addRequired(MInfo.vregsRequired))
1717 45629 : todo.insert(*PrI);
1718 : }
1719 : }
1720 1661447 : }
1721 :
1722 : // Check PHI instructions at the beginning of MBB. It is assumed that
1723 : // calcRegsPassed has been run so BBInfo::isLiveOut is valid.
1724 2091781 : void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {
1725 2091781 : BBInfo &MInfo = MBBInfoMap[&MBB];
1726 :
1727 : SmallPtrSet<const MachineBasicBlock*, 8> seen;
1728 2174961 : for (const MachineInstr &Phi : MBB) {
1729 : if (!Phi.isPHI())
1730 : break;
1731 83180 : seen.clear();
1732 :
1733 83180 : const MachineOperand &MODef = Phi.getOperand(0);
1734 83180 : if (!MODef.isReg() || !MODef.isDef()) {
1735 0 : report("Expected first PHI operand to be a register def", &MODef, 0);
1736 0 : continue;
1737 : }
1738 83180 : if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||
1739 166360 : MODef.isEarlyClobber() || MODef.isDebug())
1740 0 : report("Unexpected flag on PHI operand", &MODef, 0);
1741 83180 : unsigned DefReg = MODef.getReg();
1742 83180 : if (!TargetRegisterInfo::isVirtualRegister(DefReg))
1743 0 : report("Expected first PHI operand to be a virtual register", &MODef, 0);
1744 :
1745 254706 : for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {
1746 171526 : const MachineOperand &MO0 = Phi.getOperand(I);
1747 171526 : if (!MO0.isReg()) {
1748 0 : report("Expected PHI operand to be a register", &MO0, I);
1749 0 : continue;
1750 : }
1751 171526 : if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||
1752 343052 : MO0.isDebug() || MO0.isTied())
1753 0 : report("Unexpected flag on PHI operand", &MO0, I);
1754 :
1755 171526 : const MachineOperand &MO1 = Phi.getOperand(I + 1);
1756 171526 : if (!MO1.isMBB()) {
1757 0 : report("Expected PHI operand to be a basic block", &MO1, I + 1);
1758 0 : continue;
1759 : }
1760 :
1761 171526 : const MachineBasicBlock &Pre = *MO1.getMBB();
1762 171526 : if (!Pre.isSuccessor(&MBB)) {
1763 0 : report("PHI input is not a predecessor block", &MO1, I + 1);
1764 0 : continue;
1765 : }
1766 :
1767 171526 : if (MInfo.reachable) {
1768 171520 : seen.insert(&Pre);
1769 171520 : BBInfo &PrInfo = MBBInfoMap[&Pre];
1770 343025 : if (!MO0.isUndef() && PrInfo.reachable &&
1771 171505 : !PrInfo.isLiveOut(MO0.getReg()))
1772 2 : report("PHI operand is not live-out from predecessor", &MO0, I);
1773 : }
1774 : }
1775 :
1776 : // Did we see all predecessors?
1777 83180 : if (MInfo.reachable) {
1778 254588 : for (MachineBasicBlock *Pred : MBB.predecessors()) {
1779 171414 : if (!seen.count(Pred)) {
1780 0 : report("Missing PHI operand", &Phi);
1781 0 : errs() << printMBBReference(*Pred)
1782 0 : << " is a predecessor according to the CFG.\n";
1783 : }
1784 : }
1785 : }
1786 : }
1787 2091781 : }
1788 :
1789 1661447 : void MachineVerifier::visitMachineFunctionAfter() {
1790 1661447 : calcRegsPassed();
1791 :
1792 3753228 : for (const MachineBasicBlock &MBB : *MF)
1793 2091781 : checkPHIOps(MBB);
1794 :
1795 : // Now check liveness info if available
1796 1661447 : calcRegsRequired();
1797 :
1798 : // Check for killed virtual registers that should be live out.
1799 3753228 : for (const auto &MBB : *MF) {
1800 2091781 : BBInfo &MInfo = MBBInfoMap[&MBB];
1801 : for (RegSet::iterator
1802 2488887 : I = MInfo.vregsRequired.begin(), E = MInfo.vregsRequired.end(); I != E;
1803 : ++I)
1804 : if (MInfo.regsKilled.count(*I)) {
1805 0 : report("Virtual register killed in block, but needed live out.", &MBB);
1806 0 : errs() << "Virtual register " << printReg(*I)
1807 0 : << " is used after the block.\n";
1808 : }
1809 : }
1810 :
1811 3322894 : if (!MF->empty()) {
1812 1661308 : BBInfo &MInfo = MBBInfoMap[&MF->front()];
1813 : for (RegSet::iterator
1814 1661308 : I = MInfo.vregsRequired.begin(), E = MInfo.vregsRequired.end(); I != E;
1815 : ++I) {
1816 0 : report("Virtual register defs don't dominate all uses.", MF);
1817 0 : report_context_vreg(*I);
1818 : }
1819 : }
1820 :
1821 1661447 : if (LiveVars)
1822 120 : verifyLiveVariables();
1823 1661447 : if (LiveInts)
1824 197409 : verifyLiveIntervals();
1825 1661447 : }
1826 :
1827 120 : void MachineVerifier::verifyLiveVariables() {
1828 : assert(LiveVars && "Don't call verifyLiveVariables without LiveVars");
1829 663 : for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
1830 543 : unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
1831 543 : LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
1832 1316 : for (const auto &MBB : *MF) {
1833 773 : BBInfo &MInfo = MBBInfoMap[&MBB];
1834 :
1835 : // Our vregsRequired should be identical to LiveVariables' AliveBlocks
1836 : if (MInfo.vregsRequired.count(Reg)) {
1837 1 : if (!VI.AliveBlocks.test(MBB.getNumber())) {
1838 0 : report("LiveVariables: Block missing from AliveBlocks", &MBB);
1839 0 : errs() << "Virtual register " << printReg(Reg)
1840 0 : << " must be live through the block.\n";
1841 : }
1842 : } else {
1843 772 : if (VI.AliveBlocks.test(MBB.getNumber())) {
1844 0 : report("LiveVariables: Block should not be in AliveBlocks", &MBB);
1845 0 : errs() << "Virtual register " << printReg(Reg)
1846 0 : << " is not needed live through the block.\n";
1847 : }
1848 : }
1849 : }
1850 : }
1851 120 : }
1852 :
1853 197409 : void MachineVerifier::verifyLiveIntervals() {
1854 : assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
1855 4786529 : for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
1856 : unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
1857 :
1858 : // Spilling and splitting may leave unused registers around. Skip them.
1859 4589120 : if (MRI->reg_nodbg_empty(Reg))
1860 : continue;
1861 :
1862 1631768 : if (!LiveInts->hasInterval(Reg)) {
1863 0 : report("Missing live interval for virtual register", MF);
1864 0 : errs() << printReg(Reg, TRI) << " still has defs or uses\n";
1865 0 : continue;
1866 : }
1867 :
1868 1631768 : const LiveInterval &LI = LiveInts->getInterval(Reg);
1869 : assert(Reg == LI.reg && "Invalid reg to interval mapping");
1870 1631768 : verifyLiveInterval(LI);
1871 : }
1872 :
1873 : // Verify all the cached regunit intervals.
1874 66118912 : for (unsigned i = 0, e = TRI->getNumRegUnits(); i != e; ++i)
1875 131843006 : if (const LiveRange *LR = LiveInts->getCachedRegUnit(i))
1876 712728 : verifyLiveRange(*LR, i);
1877 197409 : }
1878 :
1879 3954499 : void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
1880 : const VNInfo *VNI, unsigned Reg,
1881 : LaneBitmask LaneMask) {
1882 3954499 : if (VNI->isUnused())
1883 : return;
1884 :
1885 0 : const VNInfo *DefVNI = LR.getVNInfoAt(VNI->def);
1886 :
1887 3953722 : if (!DefVNI) {
1888 0 : report("Value not live at VNInfo def and not marked unused", MF);
1889 0 : report_context(LR, Reg, LaneMask);
1890 0 : report_context(*VNI);
1891 0 : return;
1892 : }
1893 :
1894 3953722 : if (DefVNI != VNI) {
1895 0 : report("Live segment at def has different VNInfo", MF);
1896 0 : report_context(LR, Reg, LaneMask);
1897 0 : report_context(*VNI);
1898 0 : return;
1899 : }
1900 :
1901 3953722 : const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def);
1902 3953722 : if (!MBB) {
1903 0 : report("Invalid VNInfo definition index", MF);
1904 0 : report_context(LR, Reg, LaneMask);
1905 0 : report_context(*VNI);
1906 0 : return;
1907 : }
1908 :
1909 3953722 : if (VNI->isPHIDef()) {
1910 1020006 : if (VNI->def != LiveInts->getMBBStartIdx(MBB)) {
1911 0 : report("PHIDef VNInfo is not defined at MBB start", MBB);
1912 0 : report_context(LR, Reg, LaneMask);
1913 0 : report_context(*VNI);
1914 : }
1915 510003 : return;
1916 : }
1917 :
1918 : // Non-PHI def.
1919 : const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def);
1920 3443719 : if (!MI) {
1921 0 : report("No instruction at VNInfo def index", MBB);
1922 0 : report_context(LR, Reg, LaneMask);
1923 0 : report_context(*VNI);
1924 0 : return;
1925 : }
1926 :
1927 3443719 : if (Reg != 0) {
1928 : bool hasDef = false;
1929 : bool isEarlyClobber = false;
1930 17450698 : for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
1931 14027616 : if (!MOI->isReg() || !MOI->isDef())
1932 : continue;
1933 4108382 : if (TargetRegisterInfo::isVirtualRegister(Reg)) {
1934 3623503 : if (MOI->getReg() != Reg)
1935 : continue;
1936 : } else {
1937 1387838 : if (!TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) ||
1938 418080 : !TRI->hasRegUnit(MOI->getReg(), Reg))
1939 247839 : continue;
1940 : }
1941 3424090 : if (LaneMask.any() &&
1942 2113674 : (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none())
1943 : continue;
1944 : hasDef = true;
1945 3424068 : if (MOI->isEarlyClobber())
1946 : isEarlyClobber = true;
1947 : }
1948 :
1949 3423082 : if (!hasDef) {
1950 0 : report("Defining instruction does not modify register", MI);
1951 0 : report_context(LR, Reg, LaneMask);
1952 0 : report_context(*VNI);
1953 : }
1954 :
1955 : // Early clobber defs begin at USE slots, but other defs must begin at
1956 : // DEF slots.
1957 3423082 : if (isEarlyClobber) {
1958 17440 : if (!VNI->def.isEarlyClobber()) {
1959 0 : report("Early clobber def must be at an early-clobber slot", MBB);
1960 0 : report_context(LR, Reg, LaneMask);
1961 0 : report_context(*VNI);
1962 : }
1963 3405642 : } else if (!VNI->def.isRegister()) {
1964 0 : report("Non-PHI, non-early clobber def must be at a register slot", MBB);
1965 0 : report_context(LR, Reg, LaneMask);
1966 0 : report_context(*VNI);
1967 : }
1968 : }
1969 : }
1970 :
1971 3975850 : void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
1972 : const LiveRange::const_iterator I,
1973 : unsigned Reg, LaneBitmask LaneMask)
1974 : {
1975 : const LiveRange::Segment &S = *I;
1976 3975850 : const VNInfo *VNI = S.valno;
1977 : assert(VNI && "Live segment has no valno");
1978 :
1979 7951700 : if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(VNI->id)) {
1980 0 : report("Foreign valno in live segment", MF);
1981 0 : report_context(LR, Reg, LaneMask);
1982 0 : report_context(S);
1983 0 : report_context(*VNI);
1984 : }
1985 :
1986 3975850 : if (VNI->isUnused()) {
1987 0 : report("Live segment valno is marked unused", MF);
1988 0 : report_context(LR, Reg, LaneMask);
1989 0 : report_context(S);
1990 : }
1991 :
1992 3975850 : const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(S.start);
1993 3975850 : if (!MBB) {
1994 0 : report("Bad start of live segment, no basic block", MF);
1995 0 : report_context(LR, Reg, LaneMask);
1996 0 : report_context(S);
1997 3426880 : return;
1998 : }
1999 3975850 : SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB);
2000 3975850 : if (S.start != MBBStartIdx && S.start != VNI->def) {
2001 0 : report("Live segment must begin at MBB entry or valno def", MBB);
2002 0 : report_context(LR, Reg, LaneMask);
2003 0 : report_context(S);
2004 : }
2005 :
2006 : const MachineBasicBlock *EndMBB =
2007 7951700 : LiveInts->getMBBFromIndex(S.end.getPrevSlot());
2008 3975850 : if (!EndMBB) {
2009 0 : report("Bad end of live segment, no basic block", MF);
2010 0 : report_context(LR, Reg, LaneMask);
2011 0 : report_context(S);
2012 0 : return;
2013 : }
2014 :
2015 : // No more checks for live-out segments.
2016 7951700 : if (S.end == LiveInts->getMBBEndIdx(EndMBB))
2017 : return;
2018 :
2019 : // RegUnit intervals are allowed dead phis.
2020 736066 : if (!TargetRegisterInfo::isVirtualRegister(Reg) && VNI->isPHIDef() &&
2021 4364308 : S.start == VNI->def && S.end == VNI->def.getDeadSlot())
2022 : return;
2023 :
2024 : // The live segment is ending inside EndMBB
2025 : const MachineInstr *MI =
2026 : LiveInts->getInstructionFromIndex(S.end.getPrevSlot());
2027 3883836 : if (!MI) {
2028 0 : report("Live segment doesn't end at a valid instruction", EndMBB);
2029 0 : report_context(LR, Reg, LaneMask);
2030 0 : report_context(S);
2031 0 : return;
2032 : }
2033 :
2034 : // The block slot must refer to a basic block boundary.
2035 3883836 : if (S.end.isBlock()) {
2036 0 : report("Live segment ends at B slot of an instruction", EndMBB);
2037 0 : report_context(LR, Reg, LaneMask);
2038 0 : report_context(S);
2039 : }
2040 :
2041 3883836 : if (S.end.isDead()) {
2042 : // Segment ends on the dead slot.
2043 : // That means there must be a dead def.
2044 149908 : if (!SlotIndex::isSameInstr(S.start, S.end)) {
2045 0 : report("Live segment ending at dead slot spans instructions", EndMBB);
2046 0 : report_context(LR, Reg, LaneMask);
2047 0 : report_context(S);
2048 : }
2049 : }
2050 :
2051 : // A live segment can only end at an early-clobber slot if it is being
2052 : // redefined by an early-clobber def.
2053 3883836 : if (S.end.isEarlyClobber()) {
2054 10864 : if (I+1 == LR.end() || (I+1)->start != S.end) {
2055 0 : report("Live segment ending at early clobber slot must be "
2056 : "redefined by an EC def in the same instruction", EndMBB);
2057 0 : report_context(LR, Reg, LaneMask);
2058 0 : report_context(S);
2059 : }
2060 : }
2061 :
2062 : // The following checks only apply to virtual registers. Physreg liveness
2063 : // is too weird to check.
2064 3883836 : if (TargetRegisterInfo::isVirtualRegister(Reg)) {
2065 : // A live segment can end with either a redefinition, a kill flag on a
2066 : // use, or a dead flag on a def.
2067 : bool hasRead = false;
2068 : bool hasSubRegDef = false;
2069 : bool hasDeadDef = false;
2070 20101523 : for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
2071 16952358 : if (!MOI->isReg() || MOI->getReg() != Reg)
2072 : continue;
2073 : unsigned Sub = MOI->getSubReg();
2074 979818 : LaneBitmask SLM = Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub)
2075 3281427 : : LaneBitmask::getAll();
2076 3281427 : if (MOI->isDef()) {
2077 542721 : if (Sub != 0) {
2078 : hasSubRegDef = true;
2079 : // An operand %0:sub0 reads %0:sub1..n. Invert the lane
2080 : // mask for subregister defs. Read-undef defs will be handled by
2081 : // readsReg below.
2082 : SLM = ~SLM;
2083 : }
2084 542721 : if (MOI->isDead())
2085 : hasDeadDef = true;
2086 : }
2087 3281427 : if (LaneMask.any() && (LaneMask & SLM).none())
2088 : continue;
2089 : if (MOI->readsReg())
2090 : hasRead = true;
2091 : }
2092 3149165 : if (S.end.isDead()) {
2093 : // Make sure that the corresponding machine operand for a "dead" live
2094 : // range has the dead flag. We cannot perform this check for subregister
2095 : // liveranges as partially dead values are allowed.
2096 47143 : if (LaneMask.none() && !hasDeadDef) {
2097 0 : report("Instruction ending live segment on dead slot has no dead flag",
2098 : MI);
2099 0 : report_context(LR, Reg, LaneMask);
2100 0 : report_context(S);
2101 : }
2102 : } else {
2103 3102022 : if (!hasRead) {
2104 : // When tracking subregister liveness, the main range must start new
2105 : // values on partial register writes, even if there is no read.
2106 2 : if (!MRI->shouldTrackSubRegLiveness(Reg) || LaneMask.any() ||
2107 : !hasSubRegDef) {
2108 0 : report("Instruction ending live segment doesn't read the register",
2109 : MI);
2110 0 : report_context(LR, Reg, LaneMask);
2111 0 : report_context(S);
2112 : }
2113 : }
2114 : }
2115 : }
2116 :
2117 : // Now check all the basic blocks in this live segment.
2118 3883836 : MachineFunction::const_iterator MFI = MBB->getIterator();
2119 : // Is this live segment the beginning of a non-PHIDef VN?
2120 3883836 : if (S.start == VNI->def && !VNI->isPHIDef()) {
2121 : // Not live-in to any blocks.
2122 3377838 : if (MBB == EndMBB)
2123 : return;
2124 : // Skip this block.
2125 : ++MFI;
2126 : }
2127 :
2128 : SmallVector<SlotIndex, 4> Undefs;
2129 548970 : if (LaneMask.any()) {
2130 12822 : LiveInterval &OwnerLI = LiveInts->getInterval(Reg);
2131 12822 : OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, *MRI, *Indexes);
2132 : }
2133 :
2134 : while (true) {
2135 : assert(LiveInts->isLiveInToMBB(LR, &*MFI));
2136 : // We don't know how to track physregs into a landing pad.
2137 612127 : if (!TargetRegisterInfo::isVirtualRegister(Reg) &&
2138 477682 : MFI->isEHPad()) {
2139 71 : if (&*MFI == EndMBB)
2140 : break;
2141 : ++MFI;
2142 0 : continue;
2143 : }
2144 :
2145 : // Is VNI a PHI-def in the current block?
2146 612056 : bool IsPHI = VNI->isPHIDef() &&
2147 505497 : VNI->def == LiveInts->getMBBStartIdx(&*MFI);
2148 :
2149 : // Check that VNI is live-out of all predecessors.
2150 : for (MachineBasicBlock::const_pred_iterator PI = MFI->pred_begin(),
2151 813408 : PE = MFI->pred_end(); PI != PE; ++PI) {
2152 201352 : SlotIndex PEnd = LiveInts->getMBBEndIdx(*PI);
2153 201352 : const VNInfo *PVNI = LR.getVNInfoBefore(PEnd);
2154 :
2155 : // All predecessors must have a live-out value. However for a phi
2156 : // instruction with subregister intervals
2157 : // only one of the subregisters (not necessarily the current one) needs to
2158 : // be defined.
2159 201352 : if (!PVNI && (LaneMask.none() || !IsPHI)) {
2160 0 : if (LiveRangeCalc::isJointlyDominated(*PI, Undefs, *Indexes))
2161 : continue;
2162 0 : report("Register not marked live out of predecessor", *PI);
2163 0 : report_context(LR, Reg, LaneMask);
2164 0 : report_context(*VNI);
2165 0 : errs() << " live into " << printMBBReference(*MFI) << '@'
2166 0 : << LiveInts->getMBBStartIdx(&*MFI) << ", not live before "
2167 0 : << PEnd << '\n';
2168 0 : continue;
2169 : }
2170 :
2171 : // Only PHI-defs can take different predecessor values.
2172 201352 : if (!IsPHI && PVNI != VNI) {
2173 0 : report("Different value live out of predecessor", *PI);
2174 0 : report_context(LR, Reg, LaneMask);
2175 0 : errs() << "Valno #" << PVNI->id << " live out of "
2176 0 : << printMBBReference(*(*PI)) << '@' << PEnd << "\nValno #"
2177 0 : << VNI->id << " live into " << printMBBReference(*MFI) << '@'
2178 0 : << LiveInts->getMBBStartIdx(&*MFI) << '\n';
2179 : }
2180 : }
2181 612056 : if (&*MFI == EndMBB)
2182 : break;
2183 : ++MFI;
2184 : }
2185 : }
2186 :
2187 3370988 : void MachineVerifier::verifyLiveRange(const LiveRange &LR, unsigned Reg,
2188 : LaneBitmask LaneMask) {
2189 7325487 : for (const VNInfo *VNI : LR.valnos)
2190 3954499 : verifyLiveRangeValue(LR, VNI, Reg, LaneMask);
2191 :
2192 7346838 : for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
2193 3975850 : verifyLiveRangeSegment(LR, I, Reg, LaneMask);
2194 3370988 : }
2195 :
2196 1631768 : void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
2197 1631768 : unsigned Reg = LI.reg;
2198 : assert(TargetRegisterInfo::isVirtualRegister(Reg));
2199 1631768 : verifyLiveRange(LI, Reg);
2200 :
2201 : LaneBitmask Mask;
2202 1631768 : LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
2203 2658260 : for (const LiveInterval::SubRange &SR : LI.subranges()) {
2204 1026492 : if ((Mask & SR.LaneMask).any()) {
2205 0 : report("Lane masks of sub ranges overlap in live interval", MF);
2206 0 : report_context(LI);
2207 : }
2208 3079476 : if ((SR.LaneMask & ~MaxMask).any()) {
2209 0 : report("Subrange lanemask is invalid", MF);
2210 0 : report_context(LI);
2211 : }
2212 1026492 : if (SR.empty()) {
2213 0 : report("Subrange must not be empty", MF);
2214 0 : report_context(SR, LI.reg, SR.LaneMask);
2215 : }
2216 : Mask |= SR.LaneMask;
2217 1026492 : verifyLiveRange(SR, LI.reg, SR.LaneMask);
2218 1026492 : if (!LI.covers(SR)) {
2219 0 : report("A Subrange is not covered by the main range", MF);
2220 0 : report_context(LI);
2221 : }
2222 : }
2223 :
2224 : // Check the LI only has one connected component.
2225 1631768 : ConnectedVNInfoEqClasses ConEQ(*LiveInts);
2226 1631768 : unsigned NumComp = ConEQ.Classify(LI);
2227 1631768 : if (NumComp > 1) {
2228 0 : report("Multiple connected components in live interval", MF);
2229 0 : report_context(LI);
2230 0 : for (unsigned comp = 0; comp != NumComp; ++comp) {
2231 0 : errs() << comp << ": valnos";
2232 0 : for (LiveInterval::const_vni_iterator I = LI.vni_begin(),
2233 0 : E = LI.vni_end(); I!=E; ++I)
2234 0 : if (comp == ConEQ.getEqClass(*I))
2235 0 : errs() << ' ' << (*I)->id;
2236 0 : errs() << '\n';
2237 : }
2238 : }
2239 1631768 : }
2240 :
2241 : namespace {
2242 :
2243 : // FrameSetup and FrameDestroy can have zero adjustment, so using a single
2244 : // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the
2245 : // value is zero.
2246 : // We use a bool plus an integer to capture the stack state.
2247 : struct StackStateOfBB {
2248 : StackStateOfBB() = default;
2249 : StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup) :
2250 : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),
2251 : ExitIsSetup(ExitSetup) {}
2252 :
2253 : // Can be negative, which means we are setting up a frame.
2254 : int EntryValue = 0;
2255 : int ExitValue = 0;
2256 : bool EntryIsSetup = false;
2257 : bool ExitIsSetup = false;
2258 : };
2259 :
2260 : } // end anonymous namespace
2261 :
2262 : /// Make sure on every path through the CFG, a FrameSetup <n> is always followed
2263 : /// by a FrameDestroy <n>, stack adjustments are identical on all
2264 : /// CFG edges to a merge point, and frame is destroyed at end of a return block.
2265 1661308 : void MachineVerifier::verifyStackFrame() {
2266 1661308 : unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
2267 1661308 : unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
2268 1661308 : if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
2269 30478 : return;
2270 :
2271 1630830 : SmallVector<StackStateOfBB, 8> SPState;
2272 3261660 : SPState.resize(MF->getNumBlockIDs());
2273 : df_iterator_default_set<const MachineBasicBlock*> Reachable;
2274 :
2275 : // Visit the MBBs in DFS order.
2276 : for (df_ext_iterator<const MachineFunction *,
2277 : df_iterator_default_set<const MachineBasicBlock *>>
2278 1630830 : DFI = df_ext_begin(MF, Reachable), DFE = df_ext_end(MF, Reachable);
2279 3690273 : DFI != DFE; ++DFI) {
2280 2059443 : const MachineBasicBlock *MBB = *DFI;
2281 :
2282 : StackStateOfBB BBState;
2283 : // Check the exit state of the DFS stack predecessor.
2284 2059443 : if (DFI.getPathLength() >= 2) {
2285 428613 : const MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2);
2286 : assert(Reachable.count(StackPred) &&
2287 : "DFS stack predecessor is already visited.\n");
2288 428613 : BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;
2289 428613 : BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;
2290 : BBState.ExitValue = BBState.EntryValue;
2291 : BBState.ExitIsSetup = BBState.EntryIsSetup;
2292 : }
2293 :
2294 : // Update stack state by checking contents of MBB.
2295 26340463 : for (const auto &I : *MBB) {
2296 48562040 : if (I.getOpcode() == FrameSetupOpcode) {
2297 124540 : if (BBState.ExitIsSetup)
2298 1 : report("FrameSetup is after another FrameSetup", &I);
2299 249080 : BBState.ExitValue -= TII->getFrameTotalSize(I);
2300 : BBState.ExitIsSetup = true;
2301 : }
2302 :
2303 48562040 : if (I.getOpcode() == FrameDestroyOpcode) {
2304 124540 : int Size = TII->getFrameTotalSize(I);
2305 124540 : if (!BBState.ExitIsSetup)
2306 1 : report("FrameDestroy is not after a FrameSetup", &I);
2307 124540 : int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :
2308 : BBState.ExitValue;
2309 124540 : if (BBState.ExitIsSetup && AbsSPAdj != Size) {
2310 1 : report("FrameDestroy <n> is after FrameSetup <m>", &I);
2311 2 : errs() << "FrameDestroy <" << Size << "> is after FrameSetup <"
2312 1 : << AbsSPAdj << ">.\n";
2313 : }
2314 124540 : BBState.ExitValue += Size;
2315 : BBState.ExitIsSetup = false;
2316 : }
2317 : }
2318 4118886 : SPState[MBB->getNumber()] = BBState;
2319 :
2320 : // Make sure the exit state of any predecessor is consistent with the entry
2321 : // state.
2322 : for (MachineBasicBlock::const_pred_iterator I = MBB->pred_begin(),
2323 2671578 : E = MBB->pred_end(); I != E; ++I) {
2324 612135 : if (Reachable.count(*I) &&
2325 1050940 : (SPState[(*I)->getNumber()].ExitValue != BBState.EntryValue ||
2326 525470 : SPState[(*I)->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {
2327 0 : report("The exit stack state of a predecessor is inconsistent.", MBB);
2328 0 : errs() << "Predecessor " << printMBBReference(*(*I))
2329 0 : << " has exit state (" << SPState[(*I)->getNumber()].ExitValue
2330 0 : << ", " << SPState[(*I)->getNumber()].ExitIsSetup << "), while "
2331 0 : << printMBBReference(*MBB) << " has entry state ("
2332 0 : << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";
2333 : }
2334 : }
2335 :
2336 : // Make sure the entry state of any successor is consistent with the exit
2337 : // state.
2338 : for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
2339 2671509 : E = MBB->succ_end(); I != E; ++I) {
2340 612066 : if (Reachable.count(*I) &&
2341 267614 : (SPState[(*I)->getNumber()].EntryValue != BBState.ExitValue ||
2342 133807 : SPState[(*I)->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {
2343 0 : report("The entry stack state of a successor is inconsistent.", MBB);
2344 0 : errs() << "Successor " << printMBBReference(*(*I))
2345 0 : << " has entry state (" << SPState[(*I)->getNumber()].EntryValue
2346 0 : << ", " << SPState[(*I)->getNumber()].EntryIsSetup << "), while "
2347 0 : << printMBBReference(*MBB) << " has exit state ("
2348 0 : << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";
2349 : }
2350 : }
2351 :
2352 : // Make sure a basic block with return ends with zero stack adjustment.
2353 4102413 : if (!MBB->empty() && MBB->back().isReturn()) {
2354 1650348 : if (BBState.ExitIsSetup)
2355 0 : report("A return block ends with a FrameSetup.", MBB);
2356 1650348 : if (BBState.ExitValue)
2357 0 : report("A return block ends with a nonzero stack adjustment.", MBB);
2358 : }
2359 : }
2360 : }
|
