LLVM 17.0.0git
InstructionSelectorImpl.h
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file declares the API for the instruction selector.
10/// This class is responsible for selecting machine instructions.
11/// It's implemented by the target. It's used by the InstructionSelect pass.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DataLayout.h"
31#include "llvm/Support/Debug.h"
34#include <cassert>
35#include <cstddef>
36#include <cstdint>
37
38namespace llvm {
39
40/// GlobalISel PatFrag Predicates
41enum {
46};
47
48template <class TgtInstructionSelector, class PredicateBitset,
49 class ComplexMatcherMemFn, class CustomRendererFn>
51 TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
53 &ISelInfo,
54 const int64_t *MatchTable, const TargetInstrInfo &TII,
56 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
57 CodeGenCoverage &CoverageInfo) const {
58
59 uint64_t CurrentIdx = 0;
60 SmallVector<uint64_t, 4> OnFailResumeAt;
61
62 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
63 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
64
65 const uint16_t Flags = State.MIs[0]->getFlags();
66
67 enum RejectAction { RejectAndGiveUp, RejectAndResume };
68 auto handleReject = [&]() -> RejectAction {
69 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
70 dbgs() << CurrentIdx << ": Rejected\n");
71 if (OnFailResumeAt.empty())
72 return RejectAndGiveUp;
73 CurrentIdx = OnFailResumeAt.pop_back_val();
74 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
75 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
76 << OnFailResumeAt.size() << " try-blocks remain)\n");
77 return RejectAndResume;
78 };
79
80 auto propagateFlags = [=](NewMIVector &OutMIs) {
81 for (auto MIB : OutMIs) {
82 // Set the NoFPExcept flag when no original matched instruction could
83 // raise an FP exception, but the new instruction potentially might.
84 uint16_t MIBFlags = Flags;
85 if (NoFPException && MIB->mayRaiseFPException())
86 MIBFlags |= MachineInstr::NoFPExcept;
87 MIB.setMIFlags(MIBFlags);
88 }
89
90 return true;
91 };
92
93 while (true) {
94 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
95 int64_t MatcherOpcode = MatchTable[CurrentIdx++];
96 switch (MatcherOpcode) {
97 case GIM_Try: {
98 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
99 dbgs() << CurrentIdx << ": Begin try-block\n");
100 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
101 break;
102 }
103
104 case GIM_RecordInsn:
106 int64_t NewInsnID = MatchTable[CurrentIdx++];
107 int64_t InsnID = MatchTable[CurrentIdx++];
108 int64_t OpIdx = MatchTable[CurrentIdx++];
109
110 // As an optimisation we require that MIs[0] is always the root. Refuse
111 // any attempt to modify it.
112 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
113
114 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
115 if (!MO.isReg()) {
116 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
117 dbgs() << CurrentIdx << ": Not a register\n");
118 if (handleReject() == RejectAndGiveUp)
119 return false;
120 break;
121 }
122 if (MO.getReg().isPhysical()) {
123 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
124 dbgs() << CurrentIdx << ": Is a physical register\n");
125 if (handleReject() == RejectAndGiveUp)
126 return false;
127 break;
128 }
129
130 MachineInstr *NewMI;
131 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
132 NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
133 else
134 NewMI = MRI.getVRegDef(MO.getReg());
135
136 if ((size_t)NewInsnID < State.MIs.size())
137 State.MIs[NewInsnID] = NewMI;
138 else {
139 assert((size_t)NewInsnID == State.MIs.size() &&
140 "Expected to store MIs in order");
141 State.MIs.push_back(NewMI);
142 }
143 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
144 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
145 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
146 << ")\n");
147 break;
148 }
149
150 case GIM_CheckFeatures: {
151 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
152 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
153 dbgs() << CurrentIdx
154 << ": GIM_CheckFeatures(ExpectedBitsetID="
155 << ExpectedBitsetID << ")\n");
156 if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
157 ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
158 if (handleReject() == RejectAndGiveUp)
159 return false;
160 }
161 break;
162 }
163
164 case GIM_CheckOpcode:
166 int64_t InsnID = MatchTable[CurrentIdx++];
167 int64_t Expected0 = MatchTable[CurrentIdx++];
168 int64_t Expected1 = -1;
169 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
170 Expected1 = MatchTable[CurrentIdx++];
171
172 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
173 unsigned Opcode = State.MIs[InsnID]->getOpcode();
174
175 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
176 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
177 << "], ExpectedOpcode=" << Expected0;
178 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
179 dbgs() << " || " << Expected1;
180 dbgs() << ") // Got=" << Opcode << "\n";
181 );
182
183 if (Opcode != Expected0 && Opcode != Expected1) {
184 if (handleReject() == RejectAndGiveUp)
185 return false;
186 }
187 break;
188 }
189 case GIM_SwitchOpcode: {
190 int64_t InsnID = MatchTable[CurrentIdx++];
191 int64_t LowerBound = MatchTable[CurrentIdx++];
192 int64_t UpperBound = MatchTable[CurrentIdx++];
193 int64_t Default = MatchTable[CurrentIdx++];
194
195 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
196 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
197
198 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
199 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
200 << LowerBound << ", " << UpperBound << "), Default=" << Default
201 << ", JumpTable...) // Got=" << Opcode << "\n";
202 });
203 if (Opcode < LowerBound || UpperBound <= Opcode) {
204 CurrentIdx = Default;
205 break;
206 }
207 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
208 if (!CurrentIdx) {
209 CurrentIdx = Default;
210 break;
211 }
212 OnFailResumeAt.push_back(Default);
213 break;
214 }
215
216 case GIM_SwitchType: {
217 int64_t InsnID = MatchTable[CurrentIdx++];
218 int64_t OpIdx = MatchTable[CurrentIdx++];
219 int64_t LowerBound = MatchTable[CurrentIdx++];
220 int64_t UpperBound = MatchTable[CurrentIdx++];
221 int64_t Default = MatchTable[CurrentIdx++];
222
223 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
224 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
225
226 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
227 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
228 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
229 << UpperBound << "), Default=" << Default
230 << ", JumpTable...) // Got=";
231 if (!MO.isReg())
232 dbgs() << "Not a VReg\n";
233 else
234 dbgs() << MRI.getType(MO.getReg()) << "\n";
235 });
236 if (!MO.isReg()) {
237 CurrentIdx = Default;
238 break;
239 }
240 const LLT Ty = MRI.getType(MO.getReg());
241 const auto TyI = ISelInfo.TypeIDMap.find(Ty);
242 if (TyI == ISelInfo.TypeIDMap.end()) {
243 CurrentIdx = Default;
244 break;
245 }
246 const int64_t TypeID = TyI->second;
247 if (TypeID < LowerBound || UpperBound <= TypeID) {
248 CurrentIdx = Default;
249 break;
250 }
251 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
252 if (!CurrentIdx) {
253 CurrentIdx = Default;
254 break;
255 }
256 OnFailResumeAt.push_back(Default);
257 break;
258 }
259
261 int64_t InsnID = MatchTable[CurrentIdx++];
262 int64_t Expected = MatchTable[CurrentIdx++];
263 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
264 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
265 << InsnID << "], Expected=" << Expected << ")\n");
266 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
267 if (State.MIs[InsnID]->getNumOperands() != Expected) {
268 if (handleReject() == RejectAndGiveUp)
269 return false;
270 }
271 break;
272 }
275 int64_t InsnID = MatchTable[CurrentIdx++];
276 int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate
277 ? MatchTable[CurrentIdx++]
278 : 1;
279 int64_t Predicate = MatchTable[CurrentIdx++];
280 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
281 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
282 << InsnID << "]->getOperand(" << OpIdx
283 << "), Predicate=" << Predicate << ")\n");
284 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
285 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
286 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
287 "Expected immediate operand");
288 assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
289 int64_t Value = 0;
290 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
291 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
292 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
293 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
294 else
295 llvm_unreachable("Expected Imm or CImm operand");
296
297 if (!testImmPredicate_I64(Predicate, Value))
298 if (handleReject() == RejectAndGiveUp)
299 return false;
300 break;
301 }
303 int64_t InsnID = MatchTable[CurrentIdx++];
304 int64_t Predicate = MatchTable[CurrentIdx++];
305 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
306 dbgs()
307 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
308 << InsnID << "], Predicate=" << Predicate << ")\n");
309 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
310 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
311 "Expected G_CONSTANT");
312 assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
313 APInt Value;
314 if (State.MIs[InsnID]->getOperand(1).isCImm())
315 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
316 else
317 llvm_unreachable("Expected Imm or CImm operand");
318
319 if (!testImmPredicate_APInt(Predicate, Value))
320 if (handleReject() == RejectAndGiveUp)
321 return false;
322 break;
323 }
325 int64_t InsnID = MatchTable[CurrentIdx++];
326 int64_t Predicate = MatchTable[CurrentIdx++];
327 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
328 dbgs()
329 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
330 << InsnID << "], Predicate=" << Predicate << ")\n");
331 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
332 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
333 "Expected G_FCONSTANT");
334 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
335 assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
336 APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
337
338 if (!testImmPredicate_APFloat(Predicate, Value))
339 if (handleReject() == RejectAndGiveUp)
340 return false;
341 break;
342 }
345 int64_t InsnID = MatchTable[CurrentIdx++];
346
347 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
348 dbgs() << CurrentIdx
349 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
350 << InsnID << "])\n");
351 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
352
353 const MachineInstr *MI = State.MIs[InsnID];
354 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
355 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
356 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
357
358 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
359 if (!isBuildVectorAllOnes(*MI, MRI)) {
360 if (handleReject() == RejectAndGiveUp)
361 return false;
362 }
363 } else {
364 if (!isBuildVectorAllZeros(*MI, MRI)) {
365 if (handleReject() == RejectAndGiveUp)
366 return false;
367 }
368 }
369
370 break;
371 }
373 int64_t InsnID = MatchTable[CurrentIdx++];
374 int64_t Predicate = MatchTable[CurrentIdx++];
375 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
376 dbgs()
377 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
378 << InsnID << "], Predicate=" << Predicate << ")\n");
379 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
380 assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
381
382 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID],
383 State.RecordedOperands))
384 if (handleReject() == RejectAndGiveUp)
385 return false;
386 break;
387 }
388 case GIM_CheckHasNoUse: {
389 int64_t InsnID = MatchTable[CurrentIdx++];
390
391 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
392 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
393 << InsnID << "]\n");
394
395 const MachineInstr *MI = State.MIs[InsnID];
396 assert(MI && "Used insn before defined");
397 assert(MI->getNumDefs() > 0 && "No defs");
398 const Register Res = MI->getOperand(0).getReg();
399
400 if (!MRI.use_nodbg_empty(Res)) {
401 if (handleReject() == RejectAndGiveUp)
402 return false;
403 }
404
405 break;
406 }
408 int64_t InsnID = MatchTable[CurrentIdx++];
409 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
410 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
411 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
412 << InsnID << "], " << (uint64_t)Ordering << ")\n");
413 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
414 if (!State.MIs[InsnID]->hasOneMemOperand())
415 if (handleReject() == RejectAndGiveUp)
416 return false;
417
418 for (const auto &MMO : State.MIs[InsnID]->memoperands())
419 if (MMO->getMergedOrdering() != Ordering)
420 if (handleReject() == RejectAndGiveUp)
421 return false;
422 break;
423 }
425 int64_t InsnID = MatchTable[CurrentIdx++];
426 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
427 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
428 dbgs() << CurrentIdx
429 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
430 << InsnID << "], " << (uint64_t)Ordering << ")\n");
431 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
432 if (!State.MIs[InsnID]->hasOneMemOperand())
433 if (handleReject() == RejectAndGiveUp)
434 return false;
435
436 for (const auto &MMO : State.MIs[InsnID]->memoperands())
437 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
438 if (handleReject() == RejectAndGiveUp)
439 return false;
440 break;
441 }
443 int64_t InsnID = MatchTable[CurrentIdx++];
444 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
445 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
446 dbgs() << CurrentIdx
447 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
448 << InsnID << "], " << (uint64_t)Ordering << ")\n");
449 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
450 if (!State.MIs[InsnID]->hasOneMemOperand())
451 if (handleReject() == RejectAndGiveUp)
452 return false;
453
454 for (const auto &MMO : State.MIs[InsnID]->memoperands())
455 if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
456 if (handleReject() == RejectAndGiveUp)
457 return false;
458 break;
459 }
461 int64_t InsnID = MatchTable[CurrentIdx++];
462 int64_t MMOIdx = MatchTable[CurrentIdx++];
463 // This accepts a list of possible address spaces.
464 const int NumAddrSpace = MatchTable[CurrentIdx++];
465
466 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
467 if (handleReject() == RejectAndGiveUp)
468 return false;
469 break;
470 }
471
472 // Need to still jump to the end of the list of address spaces if we find
473 // a match earlier.
474 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
475
476 const MachineMemOperand *MMO
477 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
478 const unsigned MMOAddrSpace = MMO->getAddrSpace();
479
480 bool Success = false;
481 for (int I = 0; I != NumAddrSpace; ++I) {
482 unsigned AddrSpace = MatchTable[CurrentIdx++];
484 TgtInstructionSelector::getName(),
485 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
486 << AddrSpace << '\n');
487
488 if (AddrSpace == MMOAddrSpace) {
489 Success = true;
490 break;
491 }
492 }
493
494 CurrentIdx = LastIdx;
495 if (!Success && handleReject() == RejectAndGiveUp)
496 return false;
497 break;
498 }
500 int64_t InsnID = MatchTable[CurrentIdx++];
501 int64_t MMOIdx = MatchTable[CurrentIdx++];
502 unsigned MinAlign = MatchTable[CurrentIdx++];
503
504 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
505
506 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
507 if (handleReject() == RejectAndGiveUp)
508 return false;
509 break;
510 }
511
513 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
514 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
515 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
516 << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
517 << ")->getAlignment() >= " << MinAlign << ")\n");
518 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
519 return false;
520
521 break;
522 }
524 int64_t InsnID = MatchTable[CurrentIdx++];
525 int64_t MMOIdx = MatchTable[CurrentIdx++];
526 uint64_t Size = MatchTable[CurrentIdx++];
527
528 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
529 dbgs() << CurrentIdx
530 << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
531 << "]->memoperands() + " << MMOIdx
532 << ", Size=" << Size << ")\n");
533 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
534
535 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
536 if (handleReject() == RejectAndGiveUp)
537 return false;
538 break;
539 }
540
541 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
542
543 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
544 dbgs() << MMO->getSize() << " bytes vs " << Size
545 << " bytes\n");
546 if (MMO->getSize() != Size)
547 if (handleReject() == RejectAndGiveUp)
548 return false;
549
550 break;
551 }
555 int64_t InsnID = MatchTable[CurrentIdx++];
556 int64_t MMOIdx = MatchTable[CurrentIdx++];
557 int64_t OpIdx = MatchTable[CurrentIdx++];
558
560 TgtInstructionSelector::getName(),
561 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
562 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
563 ? "EqualTo"
564 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
565 ? "GreaterThan"
566 : "LessThan")
567 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
568 << ", OpIdx=" << OpIdx << ")\n");
569 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
570
571 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
572 if (!MO.isReg()) {
573 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574 dbgs() << CurrentIdx << ": Not a register\n");
575 if (handleReject() == RejectAndGiveUp)
576 return false;
577 break;
578 }
579
580 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
581 if (handleReject() == RejectAndGiveUp)
582 return false;
583 break;
584 }
585
586 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
587
588 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
589 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
590 MMO->getSizeInBits() != Size) {
591 if (handleReject() == RejectAndGiveUp)
592 return false;
593 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
594 MMO->getSizeInBits() >= Size) {
595 if (handleReject() == RejectAndGiveUp)
596 return false;
597 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
598 MMO->getSizeInBits() <= Size)
599 if (handleReject() == RejectAndGiveUp)
600 return false;
601
602 break;
603 }
604 case GIM_CheckType: {
605 int64_t InsnID = MatchTable[CurrentIdx++];
606 int64_t OpIdx = MatchTable[CurrentIdx++];
607 int64_t TypeID = MatchTable[CurrentIdx++];
608 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
609 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
610 << "]->getOperand(" << OpIdx
611 << "), TypeID=" << TypeID << ")\n");
612 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
613 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
614 if (!MO.isReg() ||
615 MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
616 if (handleReject() == RejectAndGiveUp)
617 return false;
618 }
619 break;
620 }
622 int64_t InsnID = MatchTable[CurrentIdx++];
623 int64_t OpIdx = MatchTable[CurrentIdx++];
624 uint64_t SizeInBits = MatchTable[CurrentIdx++];
625
626 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
627 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
628 << InsnID << "]->getOperand(" << OpIdx
629 << "), SizeInBits=" << SizeInBits << ")\n");
630 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
631 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
632 const LLT Ty = MRI.getType(MO.getReg());
633
634 // iPTR must be looked up in the target.
635 if (SizeInBits == 0) {
636 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
637 const unsigned AddrSpace = Ty.getAddressSpace();
638 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
639 }
640
641 assert(SizeInBits != 0 && "Pointer size must be known");
642
643 if (MO.isReg()) {
644 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
645 if (handleReject() == RejectAndGiveUp)
646 return false;
647 } else if (handleReject() == RejectAndGiveUp)
648 return false;
649
650 break;
651 }
653 int64_t InsnID = MatchTable[CurrentIdx++];
654 int64_t OpIdx = MatchTable[CurrentIdx++];
655 uint64_t StoreIdx = MatchTable[CurrentIdx++];
656
657 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
658 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
659 << InsnID << "]->getOperand(" << OpIdx
660 << "), StoreIdx=" << StoreIdx << ")\n");
661 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
663 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
664 break;
665 }
667 int64_t InsnID = MatchTable[CurrentIdx++];
668 int64_t OpIdx = MatchTable[CurrentIdx++];
669 int64_t RCEnum = MatchTable[CurrentIdx++];
670 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
671 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
672 << InsnID << "]->getOperand(" << OpIdx
673 << "), RCEnum=" << RCEnum << ")\n");
674 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
675 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
676 if (!MO.isReg() ||
677 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
678 MRI.getType(MO.getReg())) !=
679 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
680 if (handleReject() == RejectAndGiveUp)
681 return false;
682 }
683 break;
684 }
685
687 int64_t InsnID = MatchTable[CurrentIdx++];
688 int64_t OpIdx = MatchTable[CurrentIdx++];
689 int64_t RendererID = MatchTable[CurrentIdx++];
690 int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
691 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
692 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
693 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
694 << "]->getOperand(" << OpIdx
695 << "), ComplexPredicateID=" << ComplexPredicateID
696 << ")\n");
697 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
698 // FIXME: Use std::invoke() when it's available.
699 ComplexRendererFns Renderer =
700 (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
701 State.MIs[InsnID]->getOperand(OpIdx));
702 if (Renderer)
703 State.Renderers[RendererID] = *Renderer;
704 else
705 if (handleReject() == RejectAndGiveUp)
706 return false;
707 break;
708 }
709
711 int64_t InsnID = MatchTable[CurrentIdx++];
712 int64_t OpIdx = MatchTable[CurrentIdx++];
713 int64_t Value = MatchTable[CurrentIdx++];
714 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
715 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
716 << InsnID << "]->getOperand(" << OpIdx
717 << "), Value=" << Value << ")\n");
718 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
719 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
720 if (MO.isReg()) {
721 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
722 LLT Ty = MRI.getType(MO.getReg());
724
725 if (!isOperandImmEqual(MO, Value, MRI)) {
726 if (handleReject() == RejectAndGiveUp)
727 return false;
728 }
729 } else if (handleReject() == RejectAndGiveUp)
730 return false;
731
732 break;
733 }
734
735 case GIM_CheckLiteralInt: {
736 int64_t InsnID = MatchTable[CurrentIdx++];
737 int64_t OpIdx = MatchTable[CurrentIdx++];
738 int64_t Value = MatchTable[CurrentIdx++];
739 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
740 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
741 << InsnID << "]->getOperand(" << OpIdx
742 << "), Value=" << Value << ")\n");
743 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
744 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
745 if (MO.isImm() && MO.getImm() == Value)
746 break;
747
748 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
749 break;
750
751 if (handleReject() == RejectAndGiveUp)
752 return false;
753
754 break;
755 }
756
758 int64_t InsnID = MatchTable[CurrentIdx++];
759 int64_t OpIdx = MatchTable[CurrentIdx++];
760 int64_t Value = MatchTable[CurrentIdx++];
761 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
762 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
763 << InsnID << "]->getOperand(" << OpIdx
764 << "), Value=" << Value << ")\n");
765 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
766 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
767 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
768 if (handleReject() == RejectAndGiveUp)
769 return false;
770 break;
771 }
773 int64_t InsnID = MatchTable[CurrentIdx++];
774 int64_t OpIdx = MatchTable[CurrentIdx++];
775 int64_t Value = MatchTable[CurrentIdx++];
776 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
777 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
778 << InsnID << "]->getOperand(" << OpIdx
779 << "), Value=" << Value << ")\n");
780 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
781 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
782 if (!MO.isPredicate() || MO.getPredicate() != Value)
783 if (handleReject() == RejectAndGiveUp)
784 return false;
785 break;
786 }
787 case GIM_CheckIsMBB: {
788 int64_t InsnID = MatchTable[CurrentIdx++];
789 int64_t OpIdx = MatchTable[CurrentIdx++];
790 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
791 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
792 << "]->getOperand(" << OpIdx << "))\n");
793 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
794 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
795 if (handleReject() == RejectAndGiveUp)
796 return false;
797 }
798 break;
799 }
800 case GIM_CheckIsImm: {
801 int64_t InsnID = MatchTable[CurrentIdx++];
802 int64_t OpIdx = MatchTable[CurrentIdx++];
803 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
804 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
805 << "]->getOperand(" << OpIdx << "))\n");
806 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
807 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
808 if (handleReject() == RejectAndGiveUp)
809 return false;
810 }
811 break;
812 }
814 int64_t InsnID = MatchTable[CurrentIdx++];
815 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
816 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
817 << InsnID << "])\n");
818 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
819 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
820 if (handleReject() == RejectAndGiveUp)
821 return false;
822 }
823 break;
824 }
827 int64_t InsnID = MatchTable[CurrentIdx++];
828 int64_t OpIdx = MatchTable[CurrentIdx++];
829 int64_t OtherInsnID = MatchTable[CurrentIdx++];
830 int64_t OtherOpIdx = MatchTable[CurrentIdx++];
831 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
832 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
833 << InsnID << "][" << OpIdx << "], MIs["
834 << OtherInsnID << "][" << OtherOpIdx << "])\n");
835 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
836 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
837
838 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
839 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
840
841 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
842 if (Op.isReg() && OtherOp.isReg()) {
843 MachineInstr *MI = getDefIgnoringCopies(Op.getReg(), MRI);
844 MachineInstr *OtherMI = getDefIgnoringCopies(OtherOp.getReg(), MRI);
845 if (MI && MI == OtherMI)
846 break;
847 }
848 }
849
850 if (!Op.isIdenticalTo(OtherOp)) {
851 if (handleReject() == RejectAndGiveUp)
852 return false;
853 }
854 break;
855 }
856 case GIM_Reject:
857 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
858 dbgs() << CurrentIdx << ": GIM_Reject\n");
859 if (handleReject() == RejectAndGiveUp)
860 return false;
861 break;
862
863 case GIR_MutateOpcode: {
864 int64_t OldInsnID = MatchTable[CurrentIdx++];
865 uint64_t NewInsnID = MatchTable[CurrentIdx++];
866 int64_t NewOpcode = MatchTable[CurrentIdx++];
867 if (NewInsnID >= OutMIs.size())
868 OutMIs.resize(NewInsnID + 1);
869
870 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
871 State.MIs[OldInsnID]);
872 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
873 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
874 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
875 << NewInsnID << "], MIs[" << OldInsnID << "], "
876 << NewOpcode << ")\n");
877 break;
878 }
879
880 case GIR_BuildMI: {
881 uint64_t NewInsnID = MatchTable[CurrentIdx++];
882 int64_t Opcode = MatchTable[CurrentIdx++];
883 if (NewInsnID >= OutMIs.size())
884 OutMIs.resize(NewInsnID + 1);
885
886 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
887 MIMetadata(*State.MIs[0]), TII.get(Opcode));
888 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
889 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
890 << NewInsnID << "], " << Opcode << ")\n");
891 break;
892 }
893
894 case GIR_Copy: {
895 int64_t NewInsnID = MatchTable[CurrentIdx++];
896 int64_t OldInsnID = MatchTable[CurrentIdx++];
897 int64_t OpIdx = MatchTable[CurrentIdx++];
898 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
899 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
900 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
901 dbgs()
902 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
903 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
904 break;
905 }
906
908 int64_t NewInsnID = MatchTable[CurrentIdx++];
909 int64_t OldInsnID = MatchTable[CurrentIdx++];
910 int64_t OpIdx = MatchTable[CurrentIdx++];
911 int64_t ZeroReg = MatchTable[CurrentIdx++];
912 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
913 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
914 if (isOperandImmEqual(MO, 0, MRI))
915 OutMIs[NewInsnID].addReg(ZeroReg);
916 else
917 OutMIs[NewInsnID].add(MO);
918 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
919 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
920 << NewInsnID << "], MIs[" << OldInsnID << "], "
921 << OpIdx << ", " << ZeroReg << ")\n");
922 break;
923 }
924
925 case GIR_CopySubReg: {
926 int64_t NewInsnID = MatchTable[CurrentIdx++];
927 int64_t OldInsnID = MatchTable[CurrentIdx++];
928 int64_t OpIdx = MatchTable[CurrentIdx++];
929 int64_t SubRegIdx = MatchTable[CurrentIdx++];
930 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
931 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
932 0, SubRegIdx);
933 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
934 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
935 << NewInsnID << "], MIs[" << OldInsnID << "], "
936 << OpIdx << ", " << SubRegIdx << ")\n");
937 break;
938 }
939
940 case GIR_AddImplicitDef: {
941 int64_t InsnID = MatchTable[CurrentIdx++];
942 int64_t RegNum = MatchTable[CurrentIdx++];
943 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
944 OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
945 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
946 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
947 << InsnID << "], " << RegNum << ")\n");
948 break;
949 }
950
951 case GIR_AddImplicitUse: {
952 int64_t InsnID = MatchTable[CurrentIdx++];
953 int64_t RegNum = MatchTable[CurrentIdx++];
954 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
955 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
956 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
957 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
958 << InsnID << "], " << RegNum << ")\n");
959 break;
960 }
961
962 case GIR_AddRegister: {
963 int64_t InsnID = MatchTable[CurrentIdx++];
964 int64_t RegNum = MatchTable[CurrentIdx++];
965 uint64_t RegFlags = MatchTable[CurrentIdx++];
966 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
967 OutMIs[InsnID].addReg(RegNum, RegFlags);
969 TgtInstructionSelector::getName(),
970 dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
971 << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
972 break;
973 }
974
977 int64_t InsnID = MatchTable[CurrentIdx++];
978 int64_t TempRegID = MatchTable[CurrentIdx++];
979 uint64_t TempRegFlags = MatchTable[CurrentIdx++];
980 unsigned SubReg = 0;
981 if (MatcherOpcode == GIR_AddTempSubRegister)
982 SubReg = MatchTable[CurrentIdx++];
983
984 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
985
986 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg);
987 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
988 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
989 << InsnID << "], TempRegisters[" << TempRegID
990 << "]";
991 if (SubReg)
992 dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
993 dbgs() << ", " << TempRegFlags << ")\n");
994 break;
995 }
996
997 case GIR_AddImm: {
998 int64_t InsnID = MatchTable[CurrentIdx++];
999 int64_t Imm = MatchTable[CurrentIdx++];
1000 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1001 OutMIs[InsnID].addImm(Imm);
1002 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1003 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1004 << "], " << Imm << ")\n");
1005 break;
1006 }
1007
1008 case GIR_ComplexRenderer: {
1009 int64_t InsnID = MatchTable[CurrentIdx++];
1010 int64_t RendererID = MatchTable[CurrentIdx++];
1011 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1012 for (const auto &RenderOpFn : State.Renderers[RendererID])
1013 RenderOpFn(OutMIs[InsnID]);
1014 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1015 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1016 << InsnID << "], " << RendererID << ")\n");
1017 break;
1018 }
1020 int64_t InsnID = MatchTable[CurrentIdx++];
1021 int64_t RendererID = MatchTable[CurrentIdx++];
1022 int64_t RenderOpID = MatchTable[CurrentIdx++];
1023 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1024 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1025 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1026 dbgs() << CurrentIdx
1027 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1028 << InsnID << "], " << RendererID << ", "
1029 << RenderOpID << ")\n");
1030 break;
1031 }
1033 int64_t InsnID = MatchTable[CurrentIdx++];
1034 int64_t RendererID = MatchTable[CurrentIdx++];
1035 int64_t RenderOpID = MatchTable[CurrentIdx++];
1036 int64_t SubRegIdx = MatchTable[CurrentIdx++];
1037 MachineInstrBuilder &MI = OutMIs[InsnID];
1038 assert(MI && "Attempted to add to undefined instruction");
1039 State.Renderers[RendererID][RenderOpID](MI);
1040 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1041 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1042 dbgs() << CurrentIdx
1043 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1044 << InsnID << "], " << RendererID << ", "
1045 << RenderOpID << ", " << SubRegIdx << ")\n");
1046 break;
1047 }
1048
1050 int64_t NewInsnID = MatchTable[CurrentIdx++];
1051 int64_t OldInsnID = MatchTable[CurrentIdx++];
1052 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1053 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
1054 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1055 OutMIs[NewInsnID].addImm(
1056 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1057 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1058 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1059 else
1060 llvm_unreachable("Expected Imm or CImm operand");
1061 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1062 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1063 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1064 break;
1065 }
1066
1067 // TODO: Needs a test case once we have a pattern that uses this.
1069 int64_t NewInsnID = MatchTable[CurrentIdx++];
1070 int64_t OldInsnID = MatchTable[CurrentIdx++];
1071 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1072 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
1073 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1074 OutMIs[NewInsnID].addFPImm(
1075 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1076 else
1077 llvm_unreachable("Expected FPImm operand");
1078 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1079 dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1080 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1081 break;
1082 }
1083
1084 case GIR_CustomRenderer: {
1085 int64_t InsnID = MatchTable[CurrentIdx++];
1086 int64_t OldInsnID = MatchTable[CurrentIdx++];
1087 int64_t RendererFnID = MatchTable[CurrentIdx++];
1088 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1089 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1090 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1091 << InsnID << "], MIs[" << OldInsnID << "], "
1092 << RendererFnID << ")\n");
1093 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(
1094 OutMIs[InsnID], *State.MIs[OldInsnID],
1095 -1); // Not a source operand of the old instruction.
1096 break;
1097 }
1099 int64_t InsnID = MatchTable[CurrentIdx++];
1100 int64_t OldInsnID = MatchTable[CurrentIdx++];
1101 int64_t OpIdx = MatchTable[CurrentIdx++];
1102 int64_t RendererFnID = MatchTable[CurrentIdx++];
1103 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1104
1106 TgtInstructionSelector::getName(),
1107 dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs["
1108 << InsnID << "], MIs[" << OldInsnID << "]->getOperand("
1109 << OpIdx << "), "
1110 << RendererFnID << ")\n");
1111 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
1112 *State.MIs[OldInsnID],
1113 OpIdx);
1114 break;
1115 }
1117 int64_t InsnID = MatchTable[CurrentIdx++];
1118 int64_t OpIdx = MatchTable[CurrentIdx++];
1119 int64_t RCEnum = MatchTable[CurrentIdx++];
1120 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1121 MachineInstr &I = *OutMIs[InsnID].getInstr();
1122 MachineFunction &MF = *I.getParent()->getParent();
1124 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1125 MachineOperand &MO = I.getOperand(OpIdx);
1126 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1127 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1128 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1129 << InsnID << "], " << OpIdx << ", " << RCEnum
1130 << ")\n");
1131 break;
1132 }
1133
1135 int64_t InsnID = MatchTable[CurrentIdx++];
1136 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1137 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1138 RBI);
1139 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1140 dbgs() << CurrentIdx
1141 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1142 << InsnID << "])\n");
1143 break;
1144 }
1145
1146 case GIR_MergeMemOperands: {
1147 int64_t InsnID = MatchTable[CurrentIdx++];
1148 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1149
1150 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1151 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1152 << InsnID << "]");
1153 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
1154 while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
1156 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1157 dbgs() << ", MIs[" << MergeInsnID << "]");
1158 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
1159 OutMIs[InsnID].addMemOperand(MMO);
1160 }
1161 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
1162 break;
1163 }
1164
1165 case GIR_EraseFromParent: {
1166 int64_t InsnID = MatchTable[CurrentIdx++];
1167 assert(State.MIs[InsnID] &&
1168 "Attempted to erase an undefined instruction");
1169 State.MIs[InsnID]->eraseFromParent();
1170 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1171 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1172 << InsnID << "])\n");
1173 break;
1174 }
1175
1176 case GIR_MakeTempReg: {
1177 int64_t TempRegID = MatchTable[CurrentIdx++];
1178 int64_t TypeID = MatchTable[CurrentIdx++];
1179
1180 State.TempRegisters[TempRegID] =
1181 MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1182 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1183 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1184 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1185 break;
1186 }
1187
1188 case GIR_Coverage: {
1189 int64_t RuleID = MatchTable[CurrentIdx++];
1190 CoverageInfo.setCovered(RuleID);
1191
1192 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1193 dbgs()
1194 << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1195 break;
1196 }
1197
1198 case GIR_Done:
1199 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1200 dbgs() << CurrentIdx << ": GIR_Done\n");
1201 propagateFlags(OutMIs);
1202 return true;
1203
1204 default:
1205 llvm_unreachable("Unexpected command");
1206 }
1207 }
1208}
1209
1210} // end namespace llvm
1211
1212#endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
unsigned SubReg
unsigned const MachineRegisterInfo * MRI
#define Success
This file contains the declarations for the subclasses of Constant, which represent the different fla...
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:64
uint64_t Size
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
Type::TypeID TypeID
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
@ Flags
Definition: TextStubV5.cpp:93
Class for arbitrary precision integers.
Definition: APInt.h:75
void setCovered(uint64_t RuleID)
bool equalsInt(uint64_t V) const
A helper method that can be used to determine if the constant contained within is equal to a constant...
Definition: Constants.h:171
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:406
Tagged union holding either a T or a Error.
Definition: Error.h:470
virtual bool testImmPredicate_APInt(unsigned, const APInt &) const
virtual bool testImmPredicate_I64(unsigned, int64_t) const
bool isOperandImmEqual(const MachineOperand &MO, int64_t Value, const MachineRegisterInfo &MRI) const
bool isObviouslySafeToFold(MachineInstr &MI, MachineInstr &IntoMI) const
Return true if MI can obviously be folded into IntoMI.
virtual bool testMIPredicate_MI(unsigned, const MachineInstr &, const std::array< const MachineOperand *, 3 > &Operands) const
bool executeMatchTable(TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, const ISelInfoTy< PredicateBitset, ComplexMatcherMemFn, CustomRendererFn > &ISelInfo, const int64_t *MatchTable, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, CodeGenCoverage &CoverageInfo) const
Execute a given matcher table and return true if the match was successful and false otherwise.
virtual bool testImmPredicate_APFloat(unsigned, const APFloat &) const
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:159
constexpr bool isPointer() const
Definition: LowLevelType.h:125
constexpr unsigned getAddressSpace() const
Definition: LowLevelType.h:247
Set of metadata that should be preserved when using BuildMI().
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Representation of each machine instruction.
Definition: MachineInstr.h:68
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
uint64_t getSize() const
Return the size in bytes of the memory reference.
uint64_t getSizeInBits() const
Return the size in bits of the memory reference.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
int64_t getImm() const
bool isPredicate() const
bool isIntrinsicID() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
Intrinsic::ID getIntrinsicID() const
unsigned getPredicate() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Holds all the information related to register banks.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
virtual const RegisterBank & getRegBankFromRegClass(const TargetRegisterClass &RC, LLT Ty) const
Get a register bank that covers RC.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
void resize(size_type N)
Definition: SmallVector.h:642
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
Definition: Value.h:74
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
Definition: Utils.cpp:1126
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
Definition: Utils.cpp:53
@ GIU_MergeMemOperands_EndOfList
Indicates the end of the variable-length MergeInsnID list in a GIR_MergeMemOperands opcode.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:152
@ GIR_ComplexRenderer
Render complex operands to the specified instruction.
@ GIR_ComplexSubOperandRenderer
Render sub-operands of complex operands to the specified instruction.
@ GIR_MakeTempReg
Create a new temporary register that's not constrained.
@ GIM_CheckMemorySizeEqualTo
Check the size of the memory access for the given machine memory operand.
@ GIR_Done
A successful emission.
@ GIM_RecordNamedOperand
Predicates with 'let PredicateCodeUsesOperands = 1' need to examine some named operands that will be ...
@ GIM_Try
Begin a try-block to attempt a match and jump to OnFail if it is unsuccessful.
@ GIM_CheckIsBuildVectorAllOnes
Check if this is a vector that can be treated as a vector splat constant.
@ GIM_CheckNumOperands
Check the instruction has the right number of operands.
@ GIR_ConstrainOperandRC
Constrain an instruction operand to a register class.
@ GIM_CheckI64ImmPredicate
Check an immediate predicate on the specified instruction.
@ GIR_AddImplicitDef
Add an implicit register def to the specified instruction.
@ GIM_CheckAPIntImmPredicate
Check an immediate predicate on the specified instruction via an APInt.
@ GIM_CheckHasNoUse
Check if there's no use of the first result.
@ GIM_CheckPointerToAny
Check the type of a pointer to any address space.
@ GIM_CheckMemorySizeEqualToLLT
Check the size of the memory access for the given machine memory operand against the size of an opera...
@ GIM_CheckComplexPattern
Check the operand matches a complex predicate.
@ GIR_CopyConstantAsSImm
Render a G_CONSTANT operator as a sign-extended immediate.
@ GIR_EraseFromParent
Erase from parent.
@ GIM_SwitchType
Switch over the LLT on the specified instruction operand.
@ GIR_CopySubReg
Copy an operand to the specified instruction.
@ GIR_MutateOpcode
Mutate an instruction.
@ GIM_CheckIsBuildVectorAllZeros
@ GIM_CheckAtomicOrderingOrStrongerThan
@ GIR_AddRegister
Add an register to the specified instruction.
@ GIR_AddTempSubRegister
Add a temporary register to the specified instruction.
@ GIM_CheckIsSafeToFold
Check if the specified operand is safe to fold into the current instruction.
@ GIM_CheckOpcode
Check the opcode on the specified instruction.
@ GIM_SwitchOpcode
Switch over the opcode on the specified instruction.
@ GIM_CheckAPFloatImmPredicate
Check a floating point immediate predicate on the specified instruction.
@ GIM_Reject
Fail the current try-block, or completely fail to match if there is no current try-block.
@ GIR_AddTempRegister
Add a temporary register to the specified instruction.
@ GIR_Copy
Copy an operand to the specified instruction.
@ GIR_AddImm
Add an immediate to the specified instruction.
@ GIR_CopyFConstantAsFPImm
Render a G_FCONSTANT operator as a sign-extended immediate.
@ GIR_CopyOrAddZeroReg
Copy an operand to the specified instruction or add a zero register if the operand is a zero immediat...
@ GIM_CheckMemoryAlignment
Check the minimum alignment of the memory access for the given machine memory operand.
@ GIM_CheckIsSameOperand
Check the specified operands are identical.
@ GIM_CheckIsSameOperandIgnoreCopies
@ GIM_CheckIsMBB
Check the specified operand is an MBB.
@ GIM_CheckMemorySizeGreaterThanLLT
@ GIM_CheckRegBankForClass
Check the register bank for the specified operand.
@ GIM_CheckLiteralInt
Check the operand is a specific literal integer (i.e.
@ GIM_CheckMemorySizeLessThanLLT
@ GIR_BuildMI
Build a new instruction.
@ GIM_RecordInsn
Record the specified instruction.
@ GIM_CheckIsImm
Check the specified operand is an Imm.
@ GIM_CheckFeatures
Check the feature bits.
@ GIM_CheckMemoryAddressSpace
Check the address space of the memory access for the given machine memory operand.
@ GIR_CustomRenderer
Render operands to the specified instruction using a custom function.
@ GIM_CheckAtomicOrdering
Check a memory operation has the specified atomic ordering.
@ GIM_CheckType
Check the type for the specified operand.
@ GIM_CheckCmpPredicate
Check the operand is a specific predicate.
@ GIM_CheckOpcodeIsEither
Check the opcode on the specified instruction, checking 2 acceptable alternatives.
@ GIR_AddImplicitUse
Add an implicit register use to the specified instruction.
@ GIR_Coverage
Increment the rule coverage counter.
@ GIR_MergeMemOperands
Merge all memory operands into instruction.
@ GIM_CheckImmOperandPredicate
Check an immediate predicate on the specified instruction.
@ GIM_CheckAtomicOrderingWeakerThan
@ GIM_CheckIntrinsicID
Check the operand is a specific intrinsic ID.
@ GIM_CheckConstantInt
Check the operand is a specific integer.
@ GIR_ComplexSubOperandSubRegRenderer
Render subregisters of suboperands of complex operands to the specified instruction.
@ GIM_RecordInsnIgnoreCopies
@ GIR_CustomOperandRenderer
Render operands to the specified instruction using a custom function, reading from a specific operand...
@ GIR_ConstrainSelectedInstOperands
Constrain an instructions operands according to the instruction description.
@ GIM_CheckCxxInsnPredicate
Check a generic C++ instruction predicate.
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition: Utils.cpp:458
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool isAtLeastOrStrongerThan(AtomicOrdering AO, AtomicOrdering Other)
bool isBuildVectorAllOnes(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
Definition: Utils.cpp:1132
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:407
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:518
@ Default
The result values are uniform if and only if all operands are uniform.
bool isStrongerThan(AtomicOrdering AO, AtomicOrdering Other)
Returns true if ao is stronger than other as defined by the AtomicOrdering lattice,...
const ComplexMatcherMemFn * ComplexPredicates
SmallDenseMap< LLT, unsigned, 64 > TypeIDMap
std::array< const MachineOperand *, 3 > RecordedOperands
Named operands that predicate with 'let PredicateCodeUsesOperands = 1' referenced in its argument lis...
std::vector< ComplexRendererFns::value_type > Renderers
DenseMap< unsigned, unsigned > TempRegisters