LLVM 22.0.0git
GIMatchTableExecutorImpl.h
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10/// function. This is implemented in a separate file because the function is
11/// quite large.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16#define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17
30#include "llvm/IR/Constants.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/Type.h"
34#include "llvm/Support/Debug.h"
36#include "llvm/Support/LEB128.h"
38#include <cassert>
39#include <cstddef>
40#include <cstdint>
41
42namespace llvm {
43
44template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45 class CustomRendererFn>
47 TgtExecutor &Exec, MatcherState &State,
49 &ExecInfo,
50 MachineIRBuilder &Builder, const uint8_t *MatchTable,
52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53 const PredicateBitset &AvailableFeatures,
54 CodeGenCoverage *CoverageInfo) const {
55
56 uint64_t CurrentIdx = 0;
57 SmallVector<uint64_t, 4> OnFailResumeAt;
58 NewMIVector OutMIs;
59
60 GISelChangeObserver *Observer = Builder.getObserver();
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint32_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtExecutor::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtExecutor::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 const auto propagateFlags = [&]() {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint32_t MIBFlags = Flags | MIB.getInstr()->getFlags();
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 if (Observer)
87 Observer->changingInstr(*MIB);
88 MIB.setMIFlags(MIBFlags);
89 if (Observer)
90 Observer->changedInstr(*MIB);
91 }
92 };
93
94 // If the index is >= 0, it's an index in the type objects generated by
95 // TableGen. If the index is <0, it's an index in the recorded types object.
96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97 if (Idx >= 0)
98 return ExecInfo.TypeObjects[Idx];
99 return State.RecordedTypes[1 - Idx];
100 };
101
102 const auto readULEB = [&]() {
103 return fastDecodeULEB128(MatchTable, CurrentIdx);
104 };
105
106 // Convenience function to return a signed value. This avoids
107 // us forgetting to first cast to int8_t before casting to a
108 // wider signed int type.
109 // if we casted uint8 directly to a wider type we'd lose
110 // negative values.
111 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
112
113 const auto readU16 = [&]() {
114 auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx);
115 CurrentIdx += 2;
116 return V;
117 };
118
119 const auto readU32 = [&]() {
120 auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx);
121 CurrentIdx += 4;
122 return V;
123 };
124
125 const auto readU64 = [&]() {
126 auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx);
127 CurrentIdx += 8;
128 return V;
129 };
130
131 const auto eraseImpl = [&](MachineInstr *MI) {
132 // If we're erasing the insertion point, ensure we don't leave a dangling
133 // pointer in the builder.
134 if (Builder.getInsertPt() == MI)
135 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());
136 if (Observer)
137 Observer->erasingInstr(*MI);
138 MI->eraseFromParent();
139 };
140
141 while (true) {
142 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
143 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
144 switch (MatcherOpcode) {
145 case GIM_Try: {
146 DEBUG_WITH_TYPE(TgtExecutor::getName(),
147 dbgs() << CurrentIdx << ": Begin try-block\n");
148 OnFailResumeAt.push_back(readU32());
149 break;
150 }
151
152 case GIM_RecordInsn:
154 uint64_t NewInsnID = readULEB();
155 uint64_t InsnID = readULEB();
156 uint64_t OpIdx = readULEB();
157
158 // As an optimisation we require that MIs[0] is always the root. Refuse
159 // any attempt to modify it.
160 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
161
162 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
163 if (!MO.isReg()) {
164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
165 dbgs() << CurrentIdx << ": Not a register\n");
166 if (handleReject() == RejectAndGiveUp)
167 return false;
168 break;
169 }
170 if (MO.getReg().isPhysical()) {
171 DEBUG_WITH_TYPE(TgtExecutor::getName(),
172 dbgs() << CurrentIdx << ": Is a physical register\n");
173 if (handleReject() == RejectAndGiveUp)
174 return false;
175 break;
176 }
177
178 MachineInstr *NewMI;
179 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
180 NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
181 else
182 NewMI = MRI.getVRegDef(MO.getReg());
183
184 if ((size_t)NewInsnID < State.MIs.size())
185 State.MIs[NewInsnID] = NewMI;
186 else {
187 assert((size_t)NewInsnID == State.MIs.size() &&
188 "Expected to store MIs in order");
189 State.MIs.push_back(NewMI);
190 }
191 DEBUG_WITH_TYPE(TgtExecutor::getName(),
192 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
193 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
194 << ")\n");
195 break;
196 }
197
198 case GIM_CheckFeatures: {
199 uint16_t ExpectedBitsetID = readU16();
200 DEBUG_WITH_TYPE(TgtExecutor::getName(),
201 dbgs() << CurrentIdx
202 << ": GIM_CheckFeatures(ExpectedBitsetID="
203 << ExpectedBitsetID << ")\n");
204 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
205 ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
206 if (handleReject() == RejectAndGiveUp)
207 return false;
208 }
209 break;
210 }
211 case GIM_CheckOpcode:
213 uint64_t InsnID = readULEB();
214 uint16_t Expected0 = readU16();
215 uint16_t Expected1 = -1;
216 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
217 Expected1 = readU16();
218
219 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
220 unsigned Opcode = State.MIs[InsnID]->getOpcode();
221
222 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
223 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
224 << "], ExpectedOpcode=" << Expected0;
225 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
226 dbgs() << " || " << Expected1;
227 dbgs() << ") // Got=" << Opcode << "\n";
228 });
229
230 if (Opcode != Expected0 && Opcode != Expected1) {
231 if (handleReject() == RejectAndGiveUp)
232 return false;
233 }
234 break;
235 }
236 case GIM_SwitchOpcode: {
237 uint64_t InsnID = readULEB();
238 uint16_t LowerBound = readU16();
239 uint16_t UpperBound = readU16();
240 uint32_t Default = readU32();
241
242 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
243 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
244
245 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
246 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
247 << LowerBound << ", " << UpperBound << "), Default=" << Default
248 << ", JumpTable...) // Got=" << Opcode << "\n";
249 });
250 if (Opcode < LowerBound || UpperBound <= Opcode) {
251 CurrentIdx = Default;
252 break;
253 }
254 const auto EntryIdx = (Opcode - LowerBound);
255 // Each entry is 4 bytes
256 CurrentIdx =
257 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4));
258 if (!CurrentIdx) {
259 CurrentIdx = Default;
260 break;
261 }
262 OnFailResumeAt.push_back(Default);
263 break;
264 }
265
266 case GIM_SwitchType: {
267 uint64_t InsnID = readULEB();
268 uint64_t OpIdx = readULEB();
269 uint16_t LowerBound = readU16();
270 uint16_t UpperBound = readU16();
271 int64_t Default = readU32();
272
273 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
274 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
275
276 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
277 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
278 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
279 << UpperBound << "), Default=" << Default
280 << ", JumpTable...) // Got=";
281 if (!MO.isReg())
282 dbgs() << "Not a VReg\n";
283 else
284 dbgs() << MRI.getType(MO.getReg()) << "\n";
285 });
286 if (!MO.isReg()) {
287 CurrentIdx = Default;
288 break;
289 }
290 const LLT Ty = MRI.getType(MO.getReg());
291 const auto TyI = ExecInfo.TypeIDMap.find(Ty);
292 if (TyI == ExecInfo.TypeIDMap.end()) {
293 CurrentIdx = Default;
294 break;
295 }
296 const int64_t TypeID = TyI->second;
297 if (TypeID < LowerBound || UpperBound <= TypeID) {
298 CurrentIdx = Default;
299 break;
300 }
301 const auto NumEntry = (TypeID - LowerBound);
302 // Each entry is 4 bytes
303 CurrentIdx =
304 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4));
305 if (!CurrentIdx) {
306 CurrentIdx = Default;
307 break;
308 }
309 OnFailResumeAt.push_back(Default);
310 break;
311 }
312
315 uint64_t InsnID = readULEB();
316 uint64_t Expected = readULEB();
317 const bool IsLE = (MatcherOpcode == GIM_CheckNumOperandsLE);
318 DEBUG_WITH_TYPE(TgtExecutor::getName(),
319 dbgs() << CurrentIdx << ": GIM_CheckNumOperands"
320 << (IsLE ? "LE" : "GE") << "(MIs[" << InsnID
321 << "], Expected=" << Expected << ")\n");
322 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
323 const unsigned NumOps = State.MIs[InsnID]->getNumOperands();
324 if (IsLE ? (NumOps > Expected) : (NumOps < Expected)) {
325 if (handleReject() == RejectAndGiveUp)
326 return false;
327 }
328 break;
329 }
331 uint64_t InsnID = readULEB();
332 uint64_t Expected = readULEB();
333 DEBUG_WITH_TYPE(TgtExecutor::getName(),
334 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
335 << InsnID << "], Expected=" << Expected << ")\n");
336 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
337 if (State.MIs[InsnID]->getNumOperands() != Expected) {
338 if (handleReject() == RejectAndGiveUp)
339 return false;
340 }
341 break;
342 }
345 uint64_t InsnID = readULEB();
346 unsigned OpIdx =
347 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
348 uint16_t Predicate = readU16();
349 DEBUG_WITH_TYPE(TgtExecutor::getName(),
350 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
351 << InsnID << "]->getOperand(" << OpIdx
352 << "), Predicate=" << Predicate << ")\n");
353 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
354 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
355 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
356 "Expected immediate operand");
357 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
358 int64_t Value = 0;
359 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
360 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
361 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
362 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
363 else
364 llvm_unreachable("Expected Imm or CImm operand");
365
367 if (handleReject() == RejectAndGiveUp)
368 return false;
369 break;
370 }
372 uint64_t InsnID = readULEB();
373 uint16_t Predicate = readU16();
374 DEBUG_WITH_TYPE(TgtExecutor::getName(),
375 dbgs()
376 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
377 << InsnID << "], Predicate=" << Predicate << ")\n");
378 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
379 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
380 "Expected G_CONSTANT");
381 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
382 if (!State.MIs[InsnID]->getOperand(1).isCImm())
383 llvm_unreachable("Expected Imm or CImm operand");
384
385 const APInt &Value =
386 State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
388 if (handleReject() == RejectAndGiveUp)
389 return false;
390 break;
391 }
393 uint64_t InsnID = readULEB();
394 uint16_t Predicate = readU16();
395 DEBUG_WITH_TYPE(TgtExecutor::getName(),
396 dbgs()
397 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
398 << InsnID << "], Predicate=" << Predicate << ")\n");
399 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
400 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
401 "Expected G_FCONSTANT");
402 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
403 "Expected FPImm operand");
404 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
405 const APFloat &Value =
406 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
407
409 if (handleReject() == RejectAndGiveUp)
410 return false;
411 break;
412 }
414 uint64_t InsnID = readULEB();
415 uint64_t OpIdx = readULEB();
416 uint16_t Predicate = readU16();
417 DEBUG_WITH_TYPE(TgtExecutor::getName(),
418 dbgs() << CurrentIdx
419 << ": GIM_CheckLeafOperandPredicate(MIs[" << InsnID
420 << "]->getOperand(" << OpIdx
421 << "), Predicate=" << Predicate << ")\n");
422 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423 assert(State.MIs[InsnID]->getOperand(OpIdx).isReg() &&
424 "Expected register operand");
425 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
426 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
427
428 if (!testMOPredicate_MO(Predicate, MO, State))
429 if (handleReject() == RejectAndGiveUp)
430 return false;
431 break;
432 }
435 uint64_t InsnID = readULEB();
436
437 DEBUG_WITH_TYPE(TgtExecutor::getName(),
438 dbgs() << CurrentIdx
439 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
440 << InsnID << "])\n");
441 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
442
443 const MachineInstr *MI = State.MIs[InsnID];
444 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
445 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
446 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
447
448 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
449 if (!isBuildVectorAllOnes(*MI, MRI)) {
450 if (handleReject() == RejectAndGiveUp)
451 return false;
452 }
453 } else {
454 if (!isBuildVectorAllZeros(*MI, MRI)) {
455 if (handleReject() == RejectAndGiveUp)
456 return false;
457 }
458 }
459
460 break;
461 }
463 // Note: we don't check for invalid here because this is purely a hook to
464 // allow some executors (such as the combiner) to check arbitrary,
465 // contextless predicates, such as whether a rule is enabled or not.
466 uint16_t Predicate = readU16();
467 DEBUG_WITH_TYPE(TgtExecutor::getName(),
468 dbgs() << CurrentIdx
469 << ": GIM_CheckSimplePredicate(Predicate="
470 << Predicate << ")\n");
471 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
473 if (handleReject() == RejectAndGiveUp)
474 return false;
475 }
476 break;
477 }
479 uint64_t InsnID = readULEB();
480 uint16_t Predicate = readU16();
481 DEBUG_WITH_TYPE(TgtExecutor::getName(),
482 dbgs()
483 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
484 << InsnID << "], Predicate=" << Predicate << ")\n");
485 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
486 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
487
488 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
489 if (handleReject() == RejectAndGiveUp)
490 return false;
491 break;
492 }
493 case GIM_CheckHasNoUse: {
494 uint64_t InsnID = readULEB();
495
496 DEBUG_WITH_TYPE(TgtExecutor::getName(),
497 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
498 << InsnID << "]\n");
499
500 const MachineInstr *MI = State.MIs[InsnID];
501 assert(MI && "Used insn before defined");
502 assert(MI->getNumDefs() > 0 && "No defs");
503 const Register Res = MI->getOperand(0).getReg();
504
505 if (!MRI.use_nodbg_empty(Res)) {
506 if (handleReject() == RejectAndGiveUp)
507 return false;
508 }
509 break;
510 }
511 case GIM_CheckHasOneUse: {
512 uint64_t InsnID = readULEB();
513
514 DEBUG_WITH_TYPE(TgtExecutor::getName(),
515 dbgs() << CurrentIdx << ": GIM_CheckHasOneUse(MIs["
516 << InsnID << "]\n");
517
518 const MachineInstr *MI = State.MIs[InsnID];
519 assert(MI && "Used insn before defined");
520 assert(MI->getNumDefs() > 0 && "No defs");
521 const Register Res = MI->getOperand(0).getReg();
522
523 if (!MRI.hasOneNonDBGUse(Res)) {
524 if (handleReject() == RejectAndGiveUp)
525 return false;
526 }
527 break;
528 }
530 uint64_t InsnID = readULEB();
531 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
532 DEBUG_WITH_TYPE(TgtExecutor::getName(),
533 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
534 << InsnID << "], " << (uint64_t)Ordering << ")\n");
535 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
536 if (!State.MIs[InsnID]->hasOneMemOperand())
537 if (handleReject() == RejectAndGiveUp)
538 return false;
539
540 for (const auto &MMO : State.MIs[InsnID]->memoperands())
541 if (MMO->getMergedOrdering() != Ordering)
542 if (handleReject() == RejectAndGiveUp)
543 return false;
544 break;
545 }
547 uint64_t InsnID = readULEB();
548 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
549 DEBUG_WITH_TYPE(TgtExecutor::getName(),
550 dbgs() << CurrentIdx
551 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
552 << InsnID << "], " << (uint64_t)Ordering << ")\n");
553 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
554 if (!State.MIs[InsnID]->hasOneMemOperand())
555 if (handleReject() == RejectAndGiveUp)
556 return false;
557
558 for (const auto &MMO : State.MIs[InsnID]->memoperands())
559 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
560 if (handleReject() == RejectAndGiveUp)
561 return false;
562 break;
563 }
565 uint64_t InsnID = readULEB();
566 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
567 DEBUG_WITH_TYPE(TgtExecutor::getName(),
568 dbgs() << CurrentIdx
569 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
570 << InsnID << "], " << (uint64_t)Ordering << ")\n");
571 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572 if (!State.MIs[InsnID]->hasOneMemOperand())
573 if (handleReject() == RejectAndGiveUp)
574 return false;
575
576 for (const auto &MMO : State.MIs[InsnID]->memoperands())
577 if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
578 if (handleReject() == RejectAndGiveUp)
579 return false;
580 break;
581 }
583 uint64_t InsnID = readULEB();
584 uint64_t MMOIdx = readULEB();
585 // This accepts a list of possible address spaces.
586 const uint64_t NumAddrSpace = MatchTable[CurrentIdx++];
587
588 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
589 if (handleReject() == RejectAndGiveUp)
590 return false;
591 break;
592 }
593
594 // Need to still jump to the end of the list of address spaces if we find
595 // a match earlier.
596 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
597
598 const MachineMemOperand *MMO =
599 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
600 const unsigned MMOAddrSpace = MMO->getAddrSpace();
601
602 bool Success = false;
603 for (unsigned I = 0; I != NumAddrSpace; ++I) {
604 uint64_t AddrSpace = readULEB();
605 DEBUG_WITH_TYPE(TgtExecutor::getName(),
606 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
607 << AddrSpace << '\n');
608
609 if (AddrSpace == MMOAddrSpace) {
610 Success = true;
611 break;
612 }
613 }
614
615 CurrentIdx = LastIdx;
616 if (!Success && handleReject() == RejectAndGiveUp)
617 return false;
618 break;
619 }
621 uint64_t InsnID = readULEB();
622 uint64_t MMOIdx = readULEB();
623 uint64_t MinAlign = MatchTable[CurrentIdx++];
624
625 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
626
627 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
628 if (handleReject() == RejectAndGiveUp)
629 return false;
630 break;
631 }
632
633 MachineMemOperand *MMO =
634 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
635 DEBUG_WITH_TYPE(TgtExecutor::getName(),
636 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
637 << "(MIs[" << InsnID << "]->memoperands() + "
638 << MMOIdx << ")->getAlignment() >= " << MinAlign
639 << ")\n");
640 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
641 return false;
642
643 break;
644 }
646 uint64_t InsnID = readULEB();
647 uint64_t MMOIdx = readULEB();
648 uint32_t Size = readU32();
649
650 DEBUG_WITH_TYPE(TgtExecutor::getName(),
651 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
652 << InsnID << "]->memoperands() + " << MMOIdx
653 << ", Size=" << Size << ")\n");
654 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
655
656 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
657 if (handleReject() == RejectAndGiveUp)
658 return false;
659 break;
660 }
661
662 MachineMemOperand *MMO =
663 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
664
665 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
666 << " bytes vs " << Size
667 << " bytes\n");
668 if (MMO->getSize() != Size)
669 if (handleReject() == RejectAndGiveUp)
670 return false;
671
672 break;
673 }
677 uint64_t InsnID = readULEB();
678 uint64_t MMOIdx = readULEB();
679 uint64_t OpIdx = readULEB();
680
682 TgtExecutor::getName(),
683 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
684 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
685 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
686 ? "GreaterThan"
687 : "LessThan")
688 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
689 << ", OpIdx=" << OpIdx << ")\n");
690 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
691
692 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
693 if (!MO.isReg()) {
694 DEBUG_WITH_TYPE(TgtExecutor::getName(),
695 dbgs() << CurrentIdx << ": Not a register\n");
696 if (handleReject() == RejectAndGiveUp)
697 return false;
698 break;
699 }
700
701 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
702 if (handleReject() == RejectAndGiveUp)
703 return false;
704 break;
705 }
706
707 MachineMemOperand *MMO =
708 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
709
710 const TypeSize Size = MRI.getType(MO.getReg()).getSizeInBits();
711 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
712 MMO->getSizeInBits() != Size) {
713 if (handleReject() == RejectAndGiveUp)
714 return false;
715 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
717 if (handleReject() == RejectAndGiveUp)
718 return false;
719 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
721 if (handleReject() == RejectAndGiveUp)
722 return false;
723
724 break;
725 }
727 case GIM_CheckType: {
728 uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB();
729 uint64_t OpIdx = readULEB();
730 int TypeID = readS8();
731 DEBUG_WITH_TYPE(TgtExecutor::getName(),
732 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
733 << "]->getOperand(" << OpIdx
734 << "), TypeID=" << TypeID << ")\n");
735 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
736 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
737 if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) {
738 if (handleReject() == RejectAndGiveUp)
739 return false;
740 }
741 break;
742 }
744 uint64_t InsnID = readULEB();
745 uint64_t OpIdx = readULEB();
746 uint64_t SizeInBits = readULEB();
747
748 DEBUG_WITH_TYPE(TgtExecutor::getName(),
749 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
750 << InsnID << "]->getOperand(" << OpIdx
751 << "), SizeInBits=" << SizeInBits << ")\n");
752 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
753 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
754 const LLT Ty = MRI.getType(MO.getReg());
755
756 // iPTR must be looked up in the target.
757 if (SizeInBits == 0) {
758 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
759 const unsigned AddrSpace = Ty.getAddressSpace();
760 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
761 }
762
763 assert(SizeInBits != 0 && "Pointer size must be known");
764
765 if (MO.isReg()) {
766 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
767 if (handleReject() == RejectAndGiveUp)
768 return false;
769 } else if (handleReject() == RejectAndGiveUp)
770 return false;
771
772 break;
773 }
775 uint64_t InsnID = readULEB();
776 uint64_t OpIdx = readULEB();
777 uint64_t StoreIdx = readULEB();
778
779 DEBUG_WITH_TYPE(TgtExecutor::getName(),
780 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
781 << InsnID << "]->getOperand(" << OpIdx
782 << "), StoreIdx=" << StoreIdx << ")\n");
783 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
784 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
785 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
786 break;
787 }
788 case GIM_RecordRegType: {
789 uint64_t InsnID = readULEB();
790 uint64_t OpIdx = readULEB();
791 int TypeIdx = readS8();
792
793 DEBUG_WITH_TYPE(TgtExecutor::getName(),
794 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
795 << InsnID << "]->getOperand(" << OpIdx
796 << "), TypeIdx=" << TypeIdx << ")\n");
797 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
798 assert(TypeIdx < 0 && "Temp types always have negative indexes!");
799 // Indexes start at -1.
800 TypeIdx = 1 - TypeIdx;
801 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);
802 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
803 State.RecordedTypes.resize(TypeIdx + 1, LLT());
804 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg());
805 break;
806 }
807
810 uint64_t InsnID =
811 (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB();
812 uint64_t OpIdx = readULEB();
813 uint16_t RCEnum = readU16();
814 DEBUG_WITH_TYPE(TgtExecutor::getName(),
815 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
816 << InsnID << "]->getOperand(" << OpIdx
817 << "), RCEnum=" << RCEnum << ")\n");
818 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
819 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
820 if (!MO.isReg() ||
821 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
822 MRI.getType(MO.getReg())) !=
823 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
824 if (handleReject() == RejectAndGiveUp)
825 return false;
826 }
827 break;
828 }
829
831 uint64_t InsnID = readULEB();
832 uint64_t OpIdx = readULEB();
833 uint16_t RendererID = readU16();
834 uint16_t ComplexPredicateID = readU16();
835 DEBUG_WITH_TYPE(TgtExecutor::getName(),
836 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
837 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
838 << "]->getOperand(" << OpIdx
839 << "), ComplexPredicateID=" << ComplexPredicateID
840 << ")\n");
841 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
842 // FIXME: Use std::invoke() when it's available.
843 ComplexRendererFns Renderer =
844 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
845 State.MIs[InsnID]->getOperand(OpIdx));
846 if (Renderer)
847 State.Renderers[RendererID] = *Renderer;
848 else if (handleReject() == RejectAndGiveUp)
849 return false;
850 break;
851 }
852
855 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
856
857 uint64_t InsnID = readULEB();
858 uint64_t OpIdx = readULEB();
859 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
860 DEBUG_WITH_TYPE(TgtExecutor::getName(),
861 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
862 << InsnID << "]->getOperand(" << OpIdx
863 << "), Value=" << Value << ")\n");
864 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
865 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
866 if (MO.isReg()) {
867 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
868 LLT Ty = MRI.getType(MO.getReg());
869 // If the type is > 64 bits, it can't be a constant int, so we bail
870 // early because SignExtend64 will assert otherwise.
871 if (Ty.getScalarSizeInBits() > 64) {
872 if (handleReject() == RejectAndGiveUp)
873 return false;
874 break;
875 }
876
878 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) {
879 if (handleReject() == RejectAndGiveUp)
880 return false;
881 }
882 } else if (handleReject() == RejectAndGiveUp)
883 return false;
884
885 break;
886 }
887
888 case GIM_CheckLiteralInt: {
889 uint64_t InsnID = readULEB();
890 uint64_t OpIdx = readULEB();
891 int64_t Value = readU64();
892 DEBUG_WITH_TYPE(TgtExecutor::getName(),
893 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
894 << InsnID << "]->getOperand(" << OpIdx
895 << "), Value=" << Value << ")\n");
896 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
897 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
898 if (MO.isImm() && MO.getImm() == Value)
899 break;
900
901 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
902 break;
903
904 if (handleReject() == RejectAndGiveUp)
905 return false;
906
907 break;
908 }
909
911 uint64_t InsnID = readULEB();
912 uint64_t OpIdx = readULEB();
913 uint16_t Value = readU16();
914 DEBUG_WITH_TYPE(TgtExecutor::getName(),
915 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
916 << InsnID << "]->getOperand(" << OpIdx
917 << "), Value=" << Value << ")\n");
918 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
919 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
920 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
921 if (handleReject() == RejectAndGiveUp)
922 return false;
923 break;
924 }
926 uint64_t InsnID = readULEB();
927 uint64_t OpIdx = readULEB();
928 uint16_t Value = readU16();
929 DEBUG_WITH_TYPE(TgtExecutor::getName(),
930 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
931 << InsnID << "]->getOperand(" << OpIdx
932 << "), Value=" << Value << ")\n");
933 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
934 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
935 if (!MO.isPredicate() || MO.getPredicate() != Value)
936 if (handleReject() == RejectAndGiveUp)
937 return false;
938 break;
939 }
940 case GIM_CheckIsMBB: {
941 uint64_t InsnID = readULEB();
942 uint64_t OpIdx = readULEB();
943 DEBUG_WITH_TYPE(TgtExecutor::getName(),
944 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
945 << "]->getOperand(" << OpIdx << "))\n");
946 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
947 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
948 if (handleReject() == RejectAndGiveUp)
949 return false;
950 }
951 break;
952 }
953 case GIM_CheckIsImm: {
954 uint64_t InsnID = readULEB();
955 uint64_t OpIdx = readULEB();
956 DEBUG_WITH_TYPE(TgtExecutor::getName(),
957 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
958 << "]->getOperand(" << OpIdx << "))\n");
959 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
960 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
961 if (handleReject() == RejectAndGiveUp)
962 return false;
963 }
964 break;
965 }
967 uint64_t NumInsn = MatchTable[CurrentIdx++];
968 DEBUG_WITH_TYPE(TgtExecutor::getName(),
969 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = "
970 << NumInsn << ")\n");
971 MachineInstr &Root = *State.MIs[0];
972 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) {
973 if (!isObviouslySafeToFold(*State.MIs[K], Root)) {
974 if (handleReject() == RejectAndGiveUp)
975 return false;
976 }
977 }
978 break;
979 }
982 uint64_t InsnID = readULEB();
983 uint64_t OpIdx = readULEB();
984 uint64_t OtherInsnID = readULEB();
985 uint64_t OtherOpIdx = readULEB();
986 DEBUG_WITH_TYPE(TgtExecutor::getName(),
987 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
988 << InsnID << "][" << OpIdx << "], MIs["
989 << OtherInsnID << "][" << OtherOpIdx << "])\n");
990 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
991 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
992
993 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
994 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
995
996 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
997 if (Op.isReg() && OtherOp.isReg()) {
998 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) ==
1000 break;
1001 }
1002 }
1003
1004 if (!Op.isIdenticalTo(OtherOp)) {
1005 if (handleReject() == RejectAndGiveUp)
1006 return false;
1007 }
1008 break;
1009 }
1011 uint64_t OldInsnID = readULEB();
1012 uint64_t OldOpIdx = readULEB();
1013 uint64_t NewInsnID = readULEB();
1014 uint64_t NewOpIdx = readULEB();
1015
1016 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1017 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
1018 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1019 << NewInsnID << "][" << NewOpIdx << "])\n");
1020
1021 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1022 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1023 if (!canReplaceReg(Old, New, MRI)) {
1024 if (handleReject() == RejectAndGiveUp)
1025 return false;
1026 }
1027 break;
1028 }
1029 case GIM_MIFlags: {
1030 uint64_t InsnID = readULEB();
1031 uint32_t Flags = readU32();
1032
1033 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1034 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
1035 << "], " << Flags << ")\n");
1036 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
1037 if (handleReject() == RejectAndGiveUp)
1038 return false;
1039 }
1040 break;
1041 }
1042 case GIM_MIFlagsNot: {
1043 uint64_t InsnID = readULEB();
1044 uint32_t Flags = readU32();
1045
1046 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1047 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
1048 << "], " << Flags << ")\n");
1049 if ((State.MIs[InsnID]->getFlags() & Flags)) {
1050 if (handleReject() == RejectAndGiveUp)
1051 return false;
1052 }
1053 break;
1054 }
1055 case GIM_Reject:
1056 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1057 dbgs() << CurrentIdx << ": GIM_Reject\n");
1058 if (handleReject() == RejectAndGiveUp)
1059 return false;
1060 break;
1061 case GIR_MutateOpcode: {
1062 uint64_t OldInsnID = readULEB();
1063 uint64_t NewInsnID = readULEB();
1064 uint16_t NewOpcode = readU16();
1065 if (NewInsnID >= OutMIs.size())
1066 OutMIs.resize(NewInsnID + 1);
1067
1068 MachineInstr *OldMI = State.MIs[OldInsnID];
1069 if (Observer)
1070 Observer->changingInstr(*OldMI);
1071 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1072 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
1073 if (Observer)
1074 Observer->changedInstr(*OldMI);
1075 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1076 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1077 << NewInsnID << "], MIs[" << OldInsnID << "], "
1078 << NewOpcode << ")\n");
1079 break;
1080 }
1081
1082 case GIR_BuildRootMI:
1083 case GIR_BuildMI: {
1084 uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB();
1085 uint16_t Opcode = readU16();
1086 if (NewInsnID >= OutMIs.size())
1087 OutMIs.resize(NewInsnID + 1);
1088
1089 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1090 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1091 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1092 << NewInsnID << "], " << Opcode << ")\n");
1093 break;
1094 }
1095
1096 case GIR_BuildConstant: {
1097 uint64_t TempRegID = readULEB();
1098 uint64_t Imm = readU64();
1099 Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
1100 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1101 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1102 << TempRegID << "], Imm=" << Imm << ")\n");
1103 break;
1104 }
1105
1106 case GIR_RootToRootCopy:
1107 case GIR_Copy: {
1108 uint64_t NewInsnID =
1109 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1110 uint64_t OldInsnID =
1111 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1112 uint64_t OpIdx = readULEB();
1113 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1114 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
1115 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1116 dbgs()
1117 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1118 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1119 break;
1120 }
1121
1122 case GIR_CopyRemaining: {
1123 uint64_t NewInsnID = readULEB();
1124 uint64_t OldInsnID = readULEB();
1125 uint64_t OpIdx = readULEB();
1126 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1127 MachineInstr &OldMI = *State.MIs[OldInsnID];
1128 MachineInstrBuilder &NewMI = OutMIs[NewInsnID];
1129 for (const auto &Op : drop_begin(OldMI.operands(), OpIdx))
1130 NewMI.add(Op);
1131 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1132 dbgs() << CurrentIdx << ": GIR_CopyRemaining(OutMIs["
1133 << NewInsnID << "], MIs[" << OldInsnID
1134 << "], /*start=*/" << OpIdx << ")\n");
1135 break;
1136 }
1137
1138 case GIR_CopyOrAddZeroReg: {
1139 uint64_t NewInsnID = readULEB();
1140 uint64_t OldInsnID = readULEB();
1141 uint64_t OpIdx = readULEB();
1142 uint16_t ZeroReg = readU16();
1143 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1144 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
1145 if (isOperandImmEqual(MO, 0, MRI))
1146 OutMIs[NewInsnID].addReg(ZeroReg);
1147 else
1148 OutMIs[NewInsnID].add(MO);
1149 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1150 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1151 << NewInsnID << "], MIs[" << OldInsnID << "], "
1152 << OpIdx << ", " << ZeroReg << ")\n");
1153 break;
1154 }
1155
1156 case GIR_CopySubReg: {
1157 uint64_t NewInsnID = readULEB();
1158 uint64_t OldInsnID = readULEB();
1159 uint64_t OpIdx = readULEB();
1160 uint16_t SubRegIdx = readU16();
1161 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1162 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
1163 0, SubRegIdx);
1164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1165 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1166 << NewInsnID << "], MIs[" << OldInsnID << "], "
1167 << OpIdx << ", " << SubRegIdx << ")\n");
1168 break;
1169 }
1170
1171 case GIR_AddImplicitDef: {
1172 uint64_t InsnID = readULEB();
1173 uint16_t RegNum = readU16();
1174 uint16_t Flags = readU16();
1175 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1176 Flags |= RegState::Implicit;
1177 OutMIs[InsnID].addDef(RegNum, Flags);
1178 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1179 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1180 << InsnID << "], " << RegNum << ")\n");
1181 break;
1182 }
1183
1184 case GIR_AddImplicitUse: {
1185 uint64_t InsnID = readULEB();
1186 uint16_t RegNum = readU16();
1187 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1188 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
1189 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1190 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1191 << InsnID << "], " << RegNum << ")\n");
1192 break;
1193 }
1194
1195 case GIR_AddRegister: {
1196 uint64_t InsnID = readULEB();
1197 uint16_t RegNum = readU16();
1198 uint16_t RegFlags = readU16();
1199 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1200 OutMIs[InsnID].addReg(RegNum, RegFlags);
1201 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1202 dbgs()
1203 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1204 << "], " << RegNum << ", " << RegFlags << ")\n");
1205 break;
1206 }
1207 case GIR_AddIntrinsicID: {
1208 uint64_t InsnID = readULEB();
1209 uint16_t Value = readU16();
1210 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1211 OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
1212 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1213 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1214 << InsnID << "], " << Value << ")\n");
1215 break;
1216 }
1218 uint64_t InsnID = readULEB();
1219 uint64_t OpIdx = readULEB();
1220 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1221 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1222 << InsnID << "], OpIdx=" << OpIdx << ")\n");
1223 MachineInstr *MI = OutMIs[InsnID];
1224 assert(MI && "Modifying undefined instruction");
1225 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead();
1226 break;
1227 }
1228 case GIR_SetMIFlags: {
1229 uint64_t InsnID = readULEB();
1230 uint32_t Flags = readU32();
1231
1232 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1233 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1234 << InsnID << "], " << Flags << ")\n");
1235 MachineInstr *MI = OutMIs[InsnID];
1236 MI->setFlags(MI->getFlags() | Flags);
1237 break;
1238 }
1239 case GIR_UnsetMIFlags: {
1240 uint64_t InsnID = readULEB();
1241 uint32_t Flags = readU32();
1242
1243 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1244 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1245 << InsnID << "], " << Flags << ")\n");
1246 MachineInstr *MI = OutMIs[InsnID];
1247 MI->setFlags(MI->getFlags() & ~Flags);
1248 break;
1249 }
1250 case GIR_CopyMIFlags: {
1251 uint64_t InsnID = readULEB();
1252 uint64_t OldInsnID = readULEB();
1253
1254 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1255 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1256 << InsnID << "], MIs[" << OldInsnID << "])\n");
1257 MachineInstr *MI = OutMIs[InsnID];
1258 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1259 break;
1260 }
1264 uint64_t InsnID = readULEB();
1265 uint64_t TempRegID = readULEB();
1266 uint16_t TempRegFlags = 0;
1267 if (MatcherOpcode != GIR_AddSimpleTempRegister)
1268 TempRegFlags = readU16();
1269 uint16_t SubReg = 0;
1270 if (MatcherOpcode == GIR_AddTempSubRegister)
1271 SubReg = readU16();
1272
1273 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1274
1275 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,
1276 SubReg);
1278 TgtExecutor::getName(),
1279 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1280 << "], TempRegisters[" << TempRegID << "]";
1281 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1282 dbgs() << ", " << TempRegFlags << ")\n");
1283 break;
1284 }
1285
1286 case GIR_AddImm8:
1287 case GIR_AddImm: {
1288 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1289 uint64_t InsnID = readULEB();
1290 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1291 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1292 OutMIs[InsnID].addImm(Imm);
1293 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1294 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1295 << "], " << Imm << ")\n");
1296 break;
1297 }
1298
1299 case GIR_AddCImm: {
1300 uint64_t InsnID = readULEB();
1301 int TypeID = readS8();
1302 uint64_t Imm = readU64();
1303 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1304
1305 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1307 OutMIs[InsnID].addCImm(
1308 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true));
1309 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1310 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1311 << "], TypeID=" << TypeID << ", Imm=" << Imm
1312 << ")\n");
1313 break;
1314 }
1315
1316 case GIR_ComplexRenderer: {
1317 uint64_t InsnID = readULEB();
1318 uint16_t RendererID = readU16();
1319 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1320 for (const auto &RenderOpFn : State.Renderers[RendererID])
1321 RenderOpFn(OutMIs[InsnID]);
1322 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1323 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1324 << InsnID << "], " << RendererID << ")\n");
1325 break;
1326 }
1328 uint64_t InsnID = readULEB();
1329 uint16_t RendererID = readU16();
1330 uint64_t RenderOpID = readULEB();
1331 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1332 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1333 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1334 dbgs() << CurrentIdx
1335 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1336 << InsnID << "], " << RendererID << ", "
1337 << RenderOpID << ")\n");
1338 break;
1339 }
1341 uint64_t InsnID = readULEB();
1342 uint16_t RendererID = readU16();
1343 uint64_t RenderOpID = readULEB();
1344 uint16_t SubRegIdx = readU16();
1345 MachineInstrBuilder &MI = OutMIs[InsnID];
1346 assert(MI && "Attempted to add to undefined instruction");
1347 State.Renderers[RendererID][RenderOpID](MI);
1348 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1349 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1350 dbgs() << CurrentIdx
1351 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1352 << InsnID << "], " << RendererID << ", "
1353 << RenderOpID << ", " << SubRegIdx << ")\n");
1354 break;
1355 }
1356
1358 uint64_t NewInsnID = readULEB();
1359 uint64_t OldInsnID = readULEB();
1360 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1361 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1362 "Expected G_CONSTANT");
1363 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1364 OutMIs[NewInsnID].addImm(
1365 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1366 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1367 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1368 else
1369 llvm_unreachable("Expected Imm or CImm operand");
1370 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1371 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1372 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1373 break;
1374 }
1375
1376 // TODO: Needs a test case once we have a pattern that uses this.
1378 uint64_t NewInsnID = readULEB();
1379 uint64_t OldInsnID = readULEB();
1380 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1381 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1382 "Expected G_FCONSTANT");
1383 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1384 OutMIs[NewInsnID].addFPImm(
1385 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1386 else
1387 llvm_unreachable("Expected FPImm operand");
1388 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1389 dbgs()
1390 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1391 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1392 break;
1393 }
1394
1395 case GIR_CustomRenderer: {
1396 uint64_t InsnID = readULEB();
1397 uint64_t OldInsnID = readULEB();
1398 uint16_t RendererFnID = readU16();
1399 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1400 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1401 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1402 << InsnID << "], MIs[" << OldInsnID << "], "
1403 << RendererFnID << ")\n");
1404 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1405 OutMIs[InsnID], *State.MIs[OldInsnID],
1406 -1); // Not a source operand of the old instruction.
1407 break;
1408 }
1410 uint16_t FnID = readU16();
1411 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1412 dbgs() << CurrentIdx << ": GIR_DoneWithCustomAction(FnID="
1413 << FnID << ")\n");
1414 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1415 if (runCustomAction(FnID, State, OutMIs)) {
1416 propagateFlags();
1417 return true;
1418 }
1419
1420 if (handleReject() == RejectAndGiveUp)
1421 return false;
1422 break;
1423 }
1425 uint64_t InsnID = readULEB();
1426 uint64_t OldInsnID = readULEB();
1427 uint64_t OpIdx = readULEB();
1428 uint16_t RendererFnID = readU16();
1429 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1430
1431 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1432 dbgs() << CurrentIdx
1433 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1434 << "], MIs[" << OldInsnID << "]->getOperand("
1435 << OpIdx << "), " << RendererFnID << ")\n");
1436 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1437 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1438 break;
1439 }
1441 uint64_t InsnID = readULEB();
1442 uint64_t OpIdx = readULEB();
1443 uint16_t RCEnum = readU16();
1444 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1445 MachineInstr &I = *OutMIs[InsnID].getInstr();
1446 MachineFunction &MF = *I.getParent()->getParent();
1448 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1449 MachineOperand &MO = I.getOperand(OpIdx);
1450 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1451 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1452 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1453 << InsnID << "], " << OpIdx << ", " << RCEnum
1454 << ")\n");
1455 break;
1456 }
1457
1460 uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands)
1461 ? 0
1462 : readULEB();
1463 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1464 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1465 RBI);
1466 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1467 dbgs() << CurrentIdx
1468 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1469 << InsnID << "])\n");
1470 break;
1471 }
1472 case GIR_MergeMemOperands: {
1473 uint64_t InsnID = readULEB();
1474 uint64_t NumInsn = MatchTable[CurrentIdx++];
1475 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1476
1477 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1478 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1479 << InsnID << "]");
1480 for (unsigned K = 0; K < NumInsn; ++K) {
1481 uint64_t NextID = readULEB();
1482 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1483 dbgs() << ", MIs[" << NextID << "]");
1484 for (const auto &MMO : State.MIs[NextID]->memoperands())
1485 OutMIs[InsnID].addMemOperand(MMO);
1486 }
1487 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1488 break;
1489 }
1490 case GIR_EraseFromParent: {
1491 uint64_t InsnID = readULEB();
1492 MachineInstr *MI = State.MIs[InsnID];
1493 assert(MI && "Attempted to erase an undefined instruction");
1494 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1495 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1496 << InsnID << "])\n");
1497 eraseImpl(MI);
1498 break;
1499 }
1501 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1502 dbgs()
1503 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n");
1504 eraseImpl(State.MIs[0]);
1505 propagateFlags();
1506 return true;
1507 }
1508 case GIR_MakeTempReg: {
1509 uint64_t TempRegID = readULEB();
1510 int TypeID = readS8();
1511
1512 State.TempRegisters[TempRegID] =
1513 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));
1514 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1515 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1516 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1517 break;
1518 }
1519 case GIR_ReplaceReg: {
1520 uint64_t OldInsnID = readULEB();
1521 uint64_t OldOpIdx = readULEB();
1522 uint64_t NewInsnID = readULEB();
1523 uint64_t NewOpIdx = readULEB();
1524
1525 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1526 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1527 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1528 << NewInsnID << "][" << NewOpIdx << "])\n");
1529
1530 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1531 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1532 if (Observer)
1533 Observer->changingAllUsesOfReg(MRI, Old);
1534 MRI.replaceRegWith(Old, New);
1535 if (Observer)
1536 Observer->finishedChangingAllUsesOfReg();
1537 break;
1538 }
1540 uint64_t OldInsnID = readULEB();
1541 uint64_t OldOpIdx = readULEB();
1542 uint64_t TempRegID = readULEB();
1543
1544 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1545 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1546 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1547 << TempRegID << "])\n");
1548
1549 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1550 Register New = State.TempRegisters[TempRegID];
1551 if (Observer)
1552 Observer->changingAllUsesOfReg(MRI, Old);
1553 MRI.replaceRegWith(Old, New);
1554 if (Observer)
1555 Observer->finishedChangingAllUsesOfReg();
1556 break;
1557 }
1558 case GIR_Coverage: {
1559 uint32_t RuleID = readU32();
1561 CoverageInfo->setCovered(RuleID);
1562
1563 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1564 << ": GIR_Coverage("
1565 << RuleID << ")");
1566 break;
1567 }
1568
1569 case GIR_Done:
1570 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1571 dbgs() << CurrentIdx << ": GIR_Done\n");
1572 propagateFlags();
1573 return true;
1574 default:
1575 llvm_unreachable("Unexpected command");
1576 }
1577 }
1578}
1579
1580} // end namespace llvm
1581
1582#endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
unsigned SubReg
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineIRBuilder class.
Register const TargetRegisterInfo * TRI
Type::TypeID TypeID
MachineInstr unsigned OpIdx
This file defines the SmallVector class.
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:77
Class for arbitrary precision integers.
Definition: APInt.h:78
void setCovered(uint64_t RuleID)
bool equalsInt(uint64_t V) const
A helper method that can be used to determine if the constant contained within is equal to a constant...
Definition: Constants.h:189
This class represents an Operation in the Expression.
unsigned getPointerSizeInBits(unsigned AS=0) const
The size in bits of the pointer representation in a given address space.
Definition: DataLayout.h:390
Tagged union holding either a T or a Error.
Definition: Error.h:485
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:359
virtual bool testSimplePredicate(unsigned) const
bool executeMatchTable(TgtExecutor &Exec, MatcherState &State, const ExecInfoTy< PredicateBitset, ComplexMatcherMemFn, CustomRendererFn > &ExecInfo, MachineIRBuilder &Builder, const uint8_t *MatchTable, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, CodeGenCoverage *CoverageInfo) const
Execute a given matcher table and return true if the match was successful and false otherwise.
virtual bool testImmPredicate_APFloat(unsigned, const APFloat &) const
virtual bool testMOPredicate_MO(unsigned, const MachineOperand &, const MatcherState &State) const
virtual bool testImmPredicate_APInt(unsigned, const APInt &) const
virtual bool testMIPredicate_MI(unsigned, const MachineInstr &, const MatcherState &State) const
virtual bool testImmPredicate_I64(unsigned, int64_t) const
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t fastDecodeULEB128(const uint8_t *LLVM_ATTRIBUTE_RESTRICT MatchTable, uint64_t &CurrentIdx)
LLVM_ABI bool isOperandImmEqual(const MachineOperand &MO, int64_t Value, const MachineRegisterInfo &MRI, bool Splat=false) const
LLVM_ABI bool isObviouslySafeToFold(MachineInstr &MI, MachineInstr &IntoMI) const
Return true if MI can obviously be folded into IntoMI.
virtual bool runCustomAction(unsigned, const MatcherState &State, NewMIVector &OutMIs) const
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
LLVM_ABI void finishedChangingAllUsesOfReg()
All instructions reported as changing by changingAllUsesOfReg() have finished being changed.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
virtual void erasingInstr(MachineInstr &MI)=0
An instruction is about to be erased.
LLVM_ABI void changingAllUsesOfReg(const MachineRegisterInfo &MRI, Register Reg)
All the instructions using the given register are being changed.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:319
constexpr unsigned getScalarSizeInBits() const
Definition: LowLevelType.h:265
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:191
constexpr bool isPointer() const
Definition: LowLevelType.h:150
constexpr unsigned getAddressSpace() const
Definition: LowLevelType.h:271
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
TypeSize getValue() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
GISelChangeObserver * getObserver()
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
const MachineInstrBuilder & add(const MachineOperand &MO) const
Representation of each machine instruction.
Definition: MachineInstr.h:72
mop_range operands()
Definition: MachineInstr.h:693
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
unsigned getAddrSpace() const
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
LocationSize getSizeInBits() const
Return the size in bits of the memory reference.
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
int64_t getImm() const
bool isPredicate() const
bool isIntrinsicID() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
Intrinsic::ID getIntrinsicID() const
unsigned getPredicate() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Holds all the information related to register banks.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
virtual const RegisterBank & getRegBankFromRegClass(const TargetRegisterClass &RC, LLT Ty) const
Get a register bank that covers RC.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:78
bool empty() const
Definition: SmallVector.h:82
size_t size() const
Definition: SmallVector.h:79
void resize(size_type N)
Definition: SmallVector.h:639
void push_back(const T &Elt)
Definition: SmallVector.h:414
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1197
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
Definition: Value.h:75
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition: TypeSize.h:233
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition: TypeSize.h:240
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:338
LLVM_ABI bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
Definition: Utils.cpp:1480
LLVM_ABI Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
Definition: Utils.cpp:56
@ GICXXCustomAction_Invalid
LLVM_ABI bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:155
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition: Utils.cpp:492
constexpr T MinAlign(U A, V B)
A and B are either alignments or offsets.
Definition: MathExtras.h:362
LLVM_ABI bool canReplaceReg(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI)
Check if DstReg can be replaced with SrcReg depending on the register constraints.
Definition: Utils.cpp:201
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:207
bool isAtLeastOrStrongerThan(AtomicOrdering AO, AtomicOrdering Other)
LLVM_ABI bool isBuildVectorAllOnes(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
Definition: Utils.cpp:1486
@ Success
The lock was released successfully.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ GIR_AddIntrinsicID
Adds an intrinsic ID to the specified instruction.
@ GIR_ComplexRenderer
Render complex operands to the specified instruction.
@ GIR_ReplaceRegWithTempReg
Replaces all references to a register with a temporary register.
@ GIR_ComplexSubOperandRenderer
Render sub-operands of complex operands to the specified instruction.
@ GIR_MakeTempReg
Create a new temporary register that's not constrained.
@ GIM_CheckMemorySizeEqualTo
Check the size of the memory access for the given machine memory operand.
@ GIM_RootCheckType
GIM_CheckType but InsnID is omitted and defaults to zero.
@ GIM_RootCheckRegBankForClass
GIM_CheckRegBankForClass but InsnID is omitted and defaults to zero.
@ GIR_Done
A successful emission.
@ GIM_RecordNamedOperand
Predicates with 'let PredicateCodeUsesOperands = 1' need to examine some named operands that will be ...
@ GIM_Try
Begin a try-block to attempt a match and jump to OnFail if it is unsuccessful.
@ GIR_RootConstrainSelectedInstOperands
GIR_ConstrainSelectedInstOperands but InsnID is omitted and defaults to zero.
@ GIM_CheckIsBuildVectorAllOnes
Check if this is a vector that can be treated as a vector splat constant.
@ GIM_CheckNumOperands
Check the instruction has the right number of operands.
@ GIR_AddCImm
Add an CImm to the specified instruction.
@ GIR_ConstrainOperandRC
Constrain an instruction operand to a register class.
@ GIM_CheckI64ImmPredicate
Check an immediate predicate on the specified instruction.
@ GIR_AddImplicitDef
Add an implicit register def to the specified instruction.
@ GIM_CheckAPIntImmPredicate
Check an immediate predicate on the specified instruction via an APInt.
@ GIM_CheckHasNoUse
Check if there's no use of the first result.
@ GIM_CheckPointerToAny
Check the type of a pointer to any address space.
@ GIM_CheckMemorySizeEqualToLLT
Check the size of the memory access for the given machine memory operand against the size of an opera...
@ GIM_CheckComplexPattern
Check the operand matches a complex predicate.
@ GIR_CopyConstantAsSImm
Render a G_CONSTANT operator as a sign-extended immediate.
@ GIR_EraseFromParent
Erase from parent.
@ GIM_SwitchType
Switch over the LLT on the specified instruction operand.
@ GIR_CopySubReg
Copy an operand to the specified instruction.
@ GIR_MutateOpcode
Mutate an instruction.
@ GIM_CheckIsBuildVectorAllZeros
@ GIM_CheckAtomicOrderingOrStrongerThan
@ GIR_AddRegister
Add an register to the specified instruction.
@ GIR_AddTempSubRegister
Add a temporary register to the specified instruction.
@ GIM_CheckIsSafeToFold
Checks if the matched instructions numbered [1, 1+N) can be folded into the root (inst 0).
@ GIM_CheckOpcode
Check the opcode on the specified instruction.
@ GIR_ReplaceReg
Replaces all references to a register from an instruction with another register from another instruct...
@ GIM_SwitchOpcode
Switch over the opcode on the specified instruction.
@ GIM_CheckAPFloatImmPredicate
Check a floating point immediate predicate on the specified instruction.
@ GIM_Reject
Fail the current try-block, or completely fail to match if there is no current try-block.
@ GIR_AddSimpleTempRegister
Add a temporary register to the specified instruction without setting any flags.
@ GIR_AddTempRegister
Add a temporary register to the specified instruction.
@ GIR_Copy
Copy an operand to the specified instruction.
@ GIR_AddImm
Add an immediate to the specified instruction.
@ GIR_CopyFConstantAsFPImm
Render a G_FCONSTANT operator as a sign-extended immediate.
@ GIR_CopyRemaining
Copies all operand starting from OpIdx in OldInsnID into the new instruction NewInsnID.
@ GIM_MIFlags
Check that a matched instruction has, or doesn't have a MIFlag.
@ GIR_CopyOrAddZeroReg
Copy an operand to the specified instruction or add a zero register if the operand is a zero immediat...
@ GIM_CheckMemoryAlignment
Check the minimum alignment of the memory access for the given machine memory operand.
@ GIM_CheckIsSameOperand
Check the specified operands are identical.
@ GIR_AddImm8
Add signed 8 bit immediate to the specified instruction.
@ GIM_CheckIsSameOperandIgnoreCopies
@ GIM_CheckIsMBB
Check the specified operand is an MBB.
@ GIM_CheckNumOperandsLE
Check the instruction has a number of operands <= or >= than given number.
@ GIM_CheckMemorySizeGreaterThanLLT
@ GIM_CheckRegBankForClass
Check the register bank for the specified operand.
@ GIM_CheckNumOperandsGE
@ GIM_CheckLiteralInt
Check the operand is a specific literal integer (i.e.
@ GIM_CheckMemorySizeLessThanLLT
@ GIM_RecordRegType
Records an operand's register type into the set of temporary types.
@ GIM_CheckLeafOperandPredicate
Check a leaf predicate on the specified instruction.
@ GIM_CheckHasOneUse
Check if there's one use of the first result.
@ GIR_EraseRootFromParent_Done
Combines both a GIR_EraseFromParent 0 + GIR_Done.
@ GIR_CopyMIFlags
Copy the MIFlags of a matched instruction into an output instruction.
@ GIR_DoneWithCustomAction
Calls a C++ function that concludes the current match.
@ GIR_BuildMI
Build a new instruction.
@ GIM_RecordInsn
Record the specified instruction.
@ GIM_CheckIsImm
Check the specified operand is an Imm.
@ GIR_BuildRootMI
GIR_BuildMI but InsnID is omitted and defaults to zero.
@ GIM_CheckFeatures
Check the feature bits Feature(2) - Expected features.
@ GIM_CheckCanReplaceReg
Check we can replace all uses of a register with another.
@ GIM_CheckMemoryAddressSpace
Check the address space of the memory access for the given machine memory operand.
@ GIR_CustomRenderer
Render operands to the specified instruction using a custom function.
@ GIM_CheckAtomicOrdering
Check a memory operation has the specified atomic ordering.
@ GIM_CheckType
Check the type for the specified operand.
@ GIM_CheckConstantInt8
Check the operand is a specific 8-bit signed integer.
@ GIM_CheckCmpPredicate
Check the operand is a specific predicate.
@ GIM_CheckOpcodeIsEither
Check the opcode on the specified instruction, checking 2 acceptable alternatives.
@ GIR_SetImplicitDefDead
Marks the implicit def of a register as dead.
@ GIR_BuildConstant
Builds a constant and stores its result in a TempReg.
@ GIR_AddImplicitUse
Add an implicit register use to the specified instruction.
@ GIR_Coverage
Increment the rule coverage counter.
@ GIR_MergeMemOperands
Merge all memory operands into instruction.
@ GIM_CheckImmOperandPredicate
Check an immediate predicate on the specified instruction.
@ GIM_CheckAtomicOrderingWeakerThan
@ GIR_SetMIFlags
Set or unset a MIFlag on an instruction.
@ GIM_CheckIntrinsicID
Check the operand is a specific intrinsic ID.
@ GIM_CheckConstantInt
Check the operand is a specific integer.
@ GIR_RootToRootCopy
GIR_Copy but with both New/OldInsnIDs omitted and defaulting to zero.
@ GIR_ComplexSubOperandSubRegRenderer
Render subregisters of suboperands of complex operands to the specified instruction.
@ GIM_RecordInsnIgnoreCopies
@ GIR_CustomOperandRenderer
Render operands to the specified instruction using a custom function, reading from a specific operand...
@ GIR_ConstrainSelectedInstOperands
Constrain an instructions operands according to the instruction description.
@ GIM_CheckCxxInsnPredicate
Check a generic C++ instruction predicate.
@ GIM_CheckSimplePredicate
Check a trivial predicate which takes no arguments.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:577
LLVM_ABI Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
Definition: Utils.cpp:499
@ Default
The result values are uniform if and only if all operands are uniform.
bool isStrongerThan(AtomicOrdering AO, AtomicOrdering Other)
Returns true if ao is stronger than other as defined by the AtomicOrdering lattice,...
SmallDenseMap< LLT, unsigned, 64 > TypeIDMap
const ComplexMatcherMemFn * ComplexPredicates
std::array< const MachineOperand *, 3 > RecordedOperands
Named operands that predicate with 'let PredicateCodeUsesOperands = 1' referenced in its argument lis...
SmallVector< LLT, 4 > RecordedTypes
Types extracted from an instruction's operand.
DenseMap< unsigned, Register > TempRegisters
std::vector< ComplexRendererFns::value_type > Renderers