LLVM 19.0.0git
FastISel.cpp
Go to the documentation of this file.
1//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the FastISel class.
10//
11// "Fast" instruction selection is designed to emit very poor code quickly.
12// Also, it is not designed to be able to do much lowering, so most illegal
13// types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14// also not intended to be able to do much optimization, except in a few cases
15// where doing optimizations reduces overall compile time. For example, folding
16// constants into immediate fields is often done, because it's cheap and it
17// reduces the number of instructions later phases have to examine.
18//
19// "Fast" instruction selection is able to fail gracefully and transfer
20// control to the SelectionDAG selector for operations that it doesn't
21// support. In many cases, this allows us to avoid duplicating a lot of
22// the complicated lowering logic that SelectionDAG currently has.
23//
24// The intended use for "fast" instruction selection is "-O0" mode
25// compilation, where the quality of the generated code is irrelevant when
26// weighed against the speed at which the code can be generated. Also,
27// at -O0, the LLVM optimizers are not running, and this makes the
28// compile time of codegen a much higher portion of the overall compile
29// time. Despite its limitations, "fast" instruction selection is able to
30// handle enough code on its own to provide noticeable overall speedups
31// in -O0 compiles.
32//
33// Basic operations are supported in a target-independent way, by reading
34// the same instruction descriptions that the SelectionDAG selector reads,
35// and identifying simple arithmetic operations that can be directly selected
36// from simple operators. More complicated operations currently require
37// target-specific code.
38//
39//===----------------------------------------------------------------------===//
40
42#include "llvm/ADT/APFloat.h"
43#include "llvm/ADT/APSInt.h"
44#include "llvm/ADT/DenseMap.h"
48#include "llvm/ADT/Statistic.h"
68#include "llvm/IR/Argument.h"
69#include "llvm/IR/Attributes.h"
70#include "llvm/IR/BasicBlock.h"
71#include "llvm/IR/CallingConv.h"
72#include "llvm/IR/Constant.h"
73#include "llvm/IR/Constants.h"
74#include "llvm/IR/DataLayout.h"
75#include "llvm/IR/DebugLoc.h"
78#include "llvm/IR/Function.h"
80#include "llvm/IR/GlobalValue.h"
81#include "llvm/IR/InlineAsm.h"
82#include "llvm/IR/InstrTypes.h"
83#include "llvm/IR/Instruction.h"
86#include "llvm/IR/LLVMContext.h"
87#include "llvm/IR/Mangler.h"
88#include "llvm/IR/Metadata.h"
89#include "llvm/IR/Operator.h"
91#include "llvm/IR/Type.h"
92#include "llvm/IR/User.h"
93#include "llvm/IR/Value.h"
94#include "llvm/MC/MCContext.h"
95#include "llvm/MC/MCInstrDesc.h"
97#include "llvm/Support/Debug.h"
103#include <algorithm>
104#include <cassert>
105#include <cstdint>
106#include <iterator>
107#include <optional>
108#include <utility>
109
110using namespace llvm;
111using namespace PatternMatch;
112
113#define DEBUG_TYPE "isel"
114
115STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
116 "target-independent selector");
117STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
118 "target-specific selector");
119STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
120
121/// Set the current block to which generated machine instructions will be
122/// appended.
124 assert(LocalValueMap.empty() &&
125 "local values should be cleared after finishing a BB");
126
127 // Instructions are appended to FuncInfo.MBB. If the basic block already
128 // contains labels or copies, use the last instruction as the last local
129 // value.
130 EmitStartPt = nullptr;
131 if (!FuncInfo.MBB->empty())
134}
135
136void FastISel::finishBasicBlock() { flushLocalValueMap(); }
137
140 // Fallback to SDISel argument lowering code to deal with sret pointer
141 // parameter.
142 return false;
143
144 if (!fastLowerArguments())
145 return false;
146
147 // Enter arguments into ValueMap for uses in non-entry BBs.
149 E = FuncInfo.Fn->arg_end();
150 I != E; ++I) {
152 assert(VI != LocalValueMap.end() && "Missed an argument?");
153 FuncInfo.ValueMap[&*I] = VI->second;
154 }
155 return true;
156}
157
158/// Return the defined register if this instruction defines exactly one
159/// virtual register and uses no other virtual registers. Otherwise return 0.
161 Register RegDef;
162 for (const MachineOperand &MO : MI.operands()) {
163 if (!MO.isReg())
164 continue;
165 if (MO.isDef()) {
166 if (RegDef)
167 return Register();
168 RegDef = MO.getReg();
169 } else if (MO.getReg().isVirtual()) {
170 // This is another use of a vreg. Don't delete it.
171 return Register();
172 }
173 }
174 return RegDef;
175}
176
177static bool isRegUsedByPhiNodes(Register DefReg,
178 FunctionLoweringInfo &FuncInfo) {
179 for (auto &P : FuncInfo.PHINodesToUpdate)
180 if (P.second == DefReg)
181 return true;
182 return false;
183}
184
185void FastISel::flushLocalValueMap() {
186 // If FastISel bails out, it could leave local value instructions behind
187 // that aren't used for anything. Detect and erase those.
189 // Save the first instruction after local values, for later.
191 ++FirstNonValue;
192
195 : FuncInfo.MBB->rend();
197 for (MachineInstr &LocalMI :
199 Register DefReg = findLocalRegDef(LocalMI);
200 if (!DefReg)
201 continue;
202 if (FuncInfo.RegsWithFixups.count(DefReg))
203 continue;
204 bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
205 if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
206 if (EmitStartPt == &LocalMI)
208 LLVM_DEBUG(dbgs() << "removing dead local value materialization"
209 << LocalMI);
210 LocalMI.eraseFromParent();
211 }
212 }
213
214 if (FirstNonValue != FuncInfo.MBB->end()) {
215 // See if there are any local value instructions left. If so, we want to
216 // make sure the first one has a debug location; if it doesn't, use the
217 // first non-value instruction's debug location.
218
219 // If EmitStartPt is non-null, this block had copies at the top before
220 // FastISel started doing anything; it points to the last one, so the
221 // first local value instruction is the one after EmitStartPt.
222 // If EmitStartPt is null, the first local value instruction is at the
223 // top of the block.
224 MachineBasicBlock::iterator FirstLocalValue =
226 : FuncInfo.MBB->begin();
227 if (FirstLocalValue != FirstNonValue && !FirstLocalValue->getDebugLoc())
228 FirstLocalValue->setDebugLoc(FirstNonValue->getDebugLoc());
229 }
230 }
231
232 LocalValueMap.clear();
235 SavedInsertPt = FuncInfo.InsertPt;
236}
237
239 EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
240 // Don't handle non-simple values in FastISel.
241 if (!RealVT.isSimple())
242 return Register();
243
244 // Ignore illegal types. We must do this before looking up the value
245 // in ValueMap because Arguments are given virtual registers regardless
246 // of whether FastISel can handle them.
247 MVT VT = RealVT.getSimpleVT();
248 if (!TLI.isTypeLegal(VT)) {
249 // Handle integer promotions, though, because they're common and easy.
250 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
251 VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
252 else
253 return Register();
254 }
255
256 // Look up the value to see if we already have a register for it.
258 if (Reg)
259 return Reg;
260
261 // In bottom-up mode, just create the virtual register which will be used
262 // to hold the value. It will be materialized later.
263 if (isa<Instruction>(V) &&
264 (!isa<AllocaInst>(V) ||
265 !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
267
268 SavePoint SaveInsertPt = enterLocalValueArea();
269
270 // Materialize the value in a register. Emit any instructions in the
271 // local value area.
272 Reg = materializeRegForValue(V, VT);
273
274 leaveLocalValueArea(SaveInsertPt);
275
276 return Reg;
277}
278
279Register FastISel::materializeConstant(const Value *V, MVT VT) {
280 Register Reg;
281 if (const auto *CI = dyn_cast<ConstantInt>(V)) {
282 if (CI->getValue().getActiveBits() <= 64)
283 Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
284 } else if (isa<AllocaInst>(V))
285 Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
286 else if (isa<ConstantPointerNull>(V))
287 // Translate this as an integer zero so that it can be
288 // local-CSE'd with actual integer zeros.
289 Reg =
291 else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
292 if (CF->isNullValue())
293 Reg = fastMaterializeFloatZero(CF);
294 else
295 // Try to emit the constant directly.
296 Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
297
298 if (!Reg) {
299 // Try to emit the constant by using an integer constant with a cast.
300 const APFloat &Flt = CF->getValueAPF();
301 EVT IntVT = TLI.getPointerTy(DL);
302 uint32_t IntBitWidth = IntVT.getSizeInBits();
303 APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
304 bool isExact;
305 (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
306 if (isExact) {
307 Register IntegerReg =
308 getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
309 if (IntegerReg)
310 Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP,
311 IntegerReg);
312 }
313 }
314 } else if (const auto *Op = dyn_cast<Operator>(V)) {
315 if (!selectOperator(Op, Op->getOpcode()))
316 if (!isa<Instruction>(Op) ||
317 !fastSelectInstruction(cast<Instruction>(Op)))
318 return 0;
320 } else if (isa<UndefValue>(V)) {
323 TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
324 }
325 return Reg;
326}
327
328/// Helper for getRegForValue. This function is called when the value isn't
329/// already available in a register and must be materialized with new
330/// instructions.
331Register FastISel::materializeRegForValue(const Value *V, MVT VT) {
333 // Give the target-specific code a try first.
334 if (isa<Constant>(V))
335 Reg = fastMaterializeConstant(cast<Constant>(V));
336
337 // If target-specific code couldn't or didn't want to handle the value, then
338 // give target-independent code a try.
339 if (!Reg)
340 Reg = materializeConstant(V, VT);
341
342 // Don't cache constant materializations in the general ValueMap.
343 // To do so would require tracking what uses they dominate.
344 if (Reg) {
347 }
348 return Reg;
349}
350
352 // Look up the value to see if we already have a register for it. We
353 // cache values defined by Instructions across blocks, and other values
354 // only locally. This is because Instructions already have the SSA
355 // def-dominates-use requirement enforced.
357 if (I != FuncInfo.ValueMap.end())
358 return I->second;
359 return LocalValueMap[V];
360}
361
362void FastISel::updateValueMap(const Value *I, Register Reg, unsigned NumRegs) {
363 if (!isa<Instruction>(I)) {
364 LocalValueMap[I] = Reg;
365 return;
366 }
367
368 Register &AssignedReg = FuncInfo.ValueMap[I];
369 if (!AssignedReg)
370 // Use the new register.
371 AssignedReg = Reg;
372 else if (Reg != AssignedReg) {
373 // Arrange for uses of AssignedReg to be replaced by uses of Reg.
374 for (unsigned i = 0; i < NumRegs; i++) {
375 FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
377 }
378
379 AssignedReg = Reg;
380 }
381}
382
385 if (!IdxN)
386 // Unhandled operand. Halt "fast" selection and bail.
387 return Register();
388
389 // If the index is smaller or larger than intptr_t, truncate or extend it.
390 MVT PtrVT = TLI.getPointerTy(DL);
391 EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
392 if (IdxVT.bitsLT(PtrVT)) {
393 IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN);
394 } else if (IdxVT.bitsGT(PtrVT)) {
395 IdxN =
396 fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN);
397 }
398 return IdxN;
399}
400
402 if (getLastLocalValue()) {
404 FuncInfo.MBB = FuncInfo.InsertPt->getParent();
406 } else
408}
409
412 assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
413 "Invalid iterator!");
414 while (I != E) {
415 if (SavedInsertPt == I)
416 SavedInsertPt = E;
417 if (EmitStartPt == I)
418 EmitStartPt = E.isValid() ? &*E : nullptr;
419 if (LastLocalValue == I)
420 LastLocalValue = E.isValid() ? &*E : nullptr;
421
422 MachineInstr *Dead = &*I;
423 ++I;
424 Dead->eraseFromParent();
425 ++NumFastIselDead;
426 }
428}
429
431 SavePoint OldInsertPt = FuncInfo.InsertPt;
433 return OldInsertPt;
434}
435
438 LastLocalValue = &*std::prev(FuncInfo.InsertPt);
439
440 // Restore the previous insert position.
441 FuncInfo.InsertPt = OldInsertPt;
442}
443
444bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
445 EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
446 if (VT == MVT::Other || !VT.isSimple())
447 // Unhandled type. Halt "fast" selection and bail.
448 return false;
449
450 // We only handle legal types. For example, on x86-32 the instruction
451 // selector contains all of the 64-bit instructions from x86-64,
452 // under the assumption that i64 won't be used if the target doesn't
453 // support it.
454 if (!TLI.isTypeLegal(VT)) {
455 // MVT::i1 is special. Allow AND, OR, or XOR because they
456 // don't require additional zeroing, which makes them easy.
457 if (VT == MVT::i1 && ISD::isBitwiseLogicOp(ISDOpcode))
458 VT = TLI.getTypeToTransformTo(I->getContext(), VT);
459 else
460 return false;
461 }
462
463 // Check if the first operand is a constant, and handle it as "ri". At -O0,
464 // we don't have anything that canonicalizes operand order.
465 if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
466 if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
467 Register Op1 = getRegForValue(I->getOperand(1));
468 if (!Op1)
469 return false;
470
471 Register ResultReg =
472 fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, CI->getZExtValue(),
473 VT.getSimpleVT());
474 if (!ResultReg)
475 return false;
476
477 // We successfully emitted code for the given LLVM Instruction.
478 updateValueMap(I, ResultReg);
479 return true;
480 }
481
482 Register Op0 = getRegForValue(I->getOperand(0));
483 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
484 return false;
485
486 // Check if the second operand is a constant and handle it appropriately.
487 if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
488 uint64_t Imm = CI->getSExtValue();
489
490 // Transform "sdiv exact X, 8" -> "sra X, 3".
491 if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
492 cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
493 Imm = Log2_64(Imm);
494 ISDOpcode = ISD::SRA;
495 }
496
497 // Transform "urem x, pow2" -> "and x, pow2-1".
498 if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
499 isPowerOf2_64(Imm)) {
500 --Imm;
501 ISDOpcode = ISD::AND;
502 }
503
504 Register ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0, Imm,
505 VT.getSimpleVT());
506 if (!ResultReg)
507 return false;
508
509 // We successfully emitted code for the given LLVM Instruction.
510 updateValueMap(I, ResultReg);
511 return true;
512 }
513
514 Register Op1 = getRegForValue(I->getOperand(1));
515 if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
516 return false;
517
518 // Now we have both operands in registers. Emit the instruction.
519 Register ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
520 ISDOpcode, Op0, Op1);
521 if (!ResultReg)
522 // Target-specific code wasn't able to find a machine opcode for
523 // the given ISD opcode and type. Halt "fast" selection and bail.
524 return false;
525
526 // We successfully emitted code for the given LLVM Instruction.
527 updateValueMap(I, ResultReg);
528 return true;
529}
530
532 Register N = getRegForValue(I->getOperand(0));
533 if (!N) // Unhandled operand. Halt "fast" selection and bail.
534 return false;
535
536 // FIXME: The code below does not handle vector GEPs. Halt "fast" selection
537 // and bail.
538 if (isa<VectorType>(I->getType()))
539 return false;
540
541 // Keep a running tab of the total offset to coalesce multiple N = N + Offset
542 // into a single N = N + TotalOffset.
543 uint64_t TotalOffs = 0;
544 // FIXME: What's a good SWAG number for MaxOffs?
545 uint64_t MaxOffs = 2048;
546 MVT VT = TLI.getPointerTy(DL);
548 GTI != E; ++GTI) {
549 const Value *Idx = GTI.getOperand();
550 if (StructType *StTy = GTI.getStructTypeOrNull()) {
551 uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
552 if (Field) {
553 // N = N + Offset
554 TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
555 if (TotalOffs >= MaxOffs) {
556 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
557 if (!N) // Unhandled operand. Halt "fast" selection and bail.
558 return false;
559 TotalOffs = 0;
560 }
561 }
562 } else {
563 // If this is a constant subscript, handle it quickly.
564 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
565 if (CI->isZero())
566 continue;
567 // N = N + Offset
568 uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
569 TotalOffs += GTI.getSequentialElementStride(DL) * IdxN;
570 if (TotalOffs >= MaxOffs) {
571 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
572 if (!N) // Unhandled operand. Halt "fast" selection and bail.
573 return false;
574 TotalOffs = 0;
575 }
576 continue;
577 }
578 if (TotalOffs) {
579 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
580 if (!N) // Unhandled operand. Halt "fast" selection and bail.
581 return false;
582 TotalOffs = 0;
583 }
584
585 // N = N + Idx * ElementSize;
586 uint64_t ElementSize = GTI.getSequentialElementStride(DL);
588 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
589 return false;
590
591 if (ElementSize != 1) {
592 IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, ElementSize, VT);
593 if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
594 return false;
595 }
596 N = fastEmit_rr(VT, VT, ISD::ADD, N, IdxN);
597 if (!N) // Unhandled operand. Halt "fast" selection and bail.
598 return false;
599 }
600 }
601 if (TotalOffs) {
602 N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT);
603 if (!N) // Unhandled operand. Halt "fast" selection and bail.
604 return false;
605 }
606
607 // We successfully emitted code for the given LLVM Instruction.
609 return true;
610}
611
612bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
613 const CallInst *CI, unsigned StartIdx) {
614 for (unsigned i = StartIdx, e = CI->arg_size(); i != e; ++i) {
615 Value *Val = CI->getArgOperand(i);
616 // Check for constants and encode them with a StackMaps::ConstantOp prefix.
617 if (const auto *C = dyn_cast<ConstantInt>(Val)) {
618 Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
619 Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
620 } else if (isa<ConstantPointerNull>(Val)) {
621 Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
623 } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
624 // Values coming from a stack location also require a special encoding,
625 // but that is added later on by the target specific frame index
626 // elimination implementation.
627 auto SI = FuncInfo.StaticAllocaMap.find(AI);
628 if (SI != FuncInfo.StaticAllocaMap.end())
630 else
631 return false;
632 } else {
634 if (!Reg)
635 return false;
636 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
637 }
638 }
639 return true;
640}
641
643 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
644 // [live variables...])
645 assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
646 "Stackmap cannot return a value.");
647
648 // The stackmap intrinsic only records the live variables (the arguments
649 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
650 // intrinsic, this won't be lowered to a function call. This means we don't
651 // have to worry about calling conventions and target-specific lowering code.
652 // Instead we perform the call lowering right here.
653 //
654 // CALLSEQ_START(0, 0...)
655 // STACKMAP(id, nbytes, ...)
656 // CALLSEQ_END(0, 0)
657 //
659
660 // Add the <id> and <numBytes> constants.
661 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
662 "Expected a constant integer.");
663 const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
664 Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
665
666 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
667 "Expected a constant integer.");
668 const auto *NumBytes =
669 cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
670 Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
671
672 // Push live variables for the stack map (skipping the first two arguments
673 // <id> and <numBytes>).
674 if (!addStackMapLiveVars(Ops, I, 2))
675 return false;
676
677 // We are not adding any register mask info here, because the stackmap doesn't
678 // clobber anything.
679
680 // Add scratch registers as implicit def and early clobber.
681 CallingConv::ID CC = I->getCallingConv();
682 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
683 for (unsigned i = 0; ScratchRegs[i]; ++i)
685 ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
686 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
687
688 // Issue CALLSEQ_START
689 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
690 auto Builder =
691 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackDown));
692 const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
693 for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
694 Builder.addImm(0);
695
696 // Issue STACKMAP.
698 TII.get(TargetOpcode::STACKMAP));
699 for (auto const &MO : Ops)
700 MIB.add(MO);
701
702 // Issue CALLSEQ_END
703 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
705 .addImm(0)
706 .addImm(0);
707
708 // Inform the Frame Information that we have a stackmap in this function.
710
711 return true;
712}
713
714/// Lower an argument list according to the target calling convention.
715///
716/// This is a helper for lowering intrinsics that follow a target calling
717/// convention or require stack pointer adjustment. Only a subset of the
718/// intrinsic's operands need to participate in the calling convention.
719bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
720 unsigned NumArgs, const Value *Callee,
721 bool ForceRetVoidTy, CallLoweringInfo &CLI) {
722 ArgListTy Args;
723 Args.reserve(NumArgs);
724
725 // Populate the argument list.
726 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
727 Value *V = CI->getOperand(ArgI);
728
729 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
730
731 ArgListEntry Entry;
732 Entry.Val = V;
733 Entry.Ty = V->getType();
734 Entry.setAttributes(CI, ArgI);
735 Args.push_back(Entry);
736 }
737
738 Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
739 : CI->getType();
740 CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
741
742 return lowerCallTo(CLI);
743}
744
746 const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
747 StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
748 SmallString<32> MangledName;
749 Mangler::getNameWithPrefix(MangledName, Target, DL);
750 MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
751 return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
752}
753
755 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
756 // i32 <numBytes>,
757 // i8* <target>,
758 // i32 <numArgs>,
759 // [Args...],
760 // [live variables...])
761 CallingConv::ID CC = I->getCallingConv();
762 bool IsAnyRegCC = CC == CallingConv::AnyReg;
763 bool HasDef = !I->getType()->isVoidTy();
764 Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts();
765
766 // Check if we can lower the return type when using anyregcc.
768 if (IsAnyRegCC && HasDef) {
769 ValueType = TLI.getSimpleValueType(DL, I->getType(), /*AllowUnknown=*/true);
770 if (ValueType == MVT::Other)
771 return false;
772 }
773
774 // Get the real number of arguments participating in the call <numArgs>
775 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
776 "Expected a constant integer.");
777 const auto *NumArgsVal =
778 cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
779 unsigned NumArgs = NumArgsVal->getZExtValue();
780
781 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
782 // This includes all meta-operands up to but not including CC.
783 unsigned NumMetaOpers = PatchPointOpers::CCPos;
784 assert(I->arg_size() >= NumMetaOpers + NumArgs &&
785 "Not enough arguments provided to the patchpoint intrinsic");
786
787 // For AnyRegCC the arguments are lowered later on manually.
788 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
790 CLI.setIsPatchPoint();
791 if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
792 return false;
793
794 assert(CLI.Call && "No call instruction specified.");
795
797
798 // Add an explicit result reg if we use the anyreg calling convention.
799 if (IsAnyRegCC && HasDef) {
800 assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
801 assert(ValueType.isValid());
803 CLI.NumResultRegs = 1;
804 Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*isDef=*/true));
805 }
806
807 // Add the <id> and <numBytes> constants.
808 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
809 "Expected a constant integer.");
810 const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
811 Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
812
813 assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
814 "Expected a constant integer.");
815 const auto *NumBytes =
816 cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
817 Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
818
819 // Add the call target.
820 if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
821 uint64_t CalleeConstAddr =
822 cast<ConstantInt>(C->getOperand(0))->getZExtValue();
823 Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
824 } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
825 if (C->getOpcode() == Instruction::IntToPtr) {
826 uint64_t CalleeConstAddr =
827 cast<ConstantInt>(C->getOperand(0))->getZExtValue();
828 Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
829 } else
830 llvm_unreachable("Unsupported ConstantExpr.");
831 } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
833 } else if (isa<ConstantPointerNull>(Callee))
835 else
836 llvm_unreachable("Unsupported callee address.");
837
838 // Adjust <numArgs> to account for any arguments that have been passed on
839 // the stack instead.
840 unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
841 Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
842
843 // Add the calling convention
844 Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
845
846 // Add the arguments we omitted previously. The register allocator should
847 // place these in any free register.
848 if (IsAnyRegCC) {
849 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
850 Register Reg = getRegForValue(I->getArgOperand(i));
851 if (!Reg)
852 return false;
853 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
854 }
855 }
856
857 // Push the arguments from the call instruction.
858 for (auto Reg : CLI.OutRegs)
859 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
860
861 // Push live variables for the stack map.
862 if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
863 return false;
864
865 // Push the register mask info.
868
869 // Add scratch registers as implicit def and early clobber.
870 const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
871 for (unsigned i = 0; ScratchRegs[i]; ++i)
873 ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
874 /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
875
876 // Add implicit defs (return values).
877 for (auto Reg : CLI.InRegs)
878 Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/true,
879 /*isImp=*/true));
880
881 // Insert the patchpoint instruction before the call generated by the target.
883 TII.get(TargetOpcode::PATCHPOINT));
884
885 for (auto &MO : Ops)
886 MIB.add(MO);
887
889
890 // Delete the original call instruction.
891 CLI.Call->eraseFromParent();
892
893 // Inform the Frame Information that we have a patchpoint in this function.
895
896 if (CLI.NumResultRegs)
898 return true;
899}
900
902 const auto &Triple = TM.getTargetTriple();
904 return true; // don't do anything to this instruction.
907 /*isDef=*/false));
909 /*isDef=*/false));
912 TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
913 for (auto &MO : Ops)
914 MIB.add(MO);
915
916 // Insert the Patchable Event Call instruction, that gets lowered properly.
917 return true;
918}
919
921 const auto &Triple = TM.getTargetTriple();
923 return true; // don't do anything to this instruction.
926 /*isDef=*/false));
928 /*isDef=*/false));
930 /*isDef=*/false));
933 TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
934 for (auto &MO : Ops)
935 MIB.add(MO);
936
937 // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
938 return true;
939}
940
941/// Returns an AttributeList representing the attributes applied to the return
942/// value of the given call.
945 if (CLI.RetSExt)
946 Attrs.push_back(Attribute::SExt);
947 if (CLI.RetZExt)
948 Attrs.push_back(Attribute::ZExt);
949 if (CLI.IsInReg)
950 Attrs.push_back(Attribute::InReg);
951
953 Attrs);
954}
955
956bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
957 unsigned NumArgs) {
958 MCContext &Ctx = MF->getContext();
959 SmallString<32> MangledName;
960 Mangler::getNameWithPrefix(MangledName, SymName, DL);
961 MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
962 return lowerCallTo(CI, Sym, NumArgs);
963}
964
966 unsigned NumArgs) {
967 FunctionType *FTy = CI->getFunctionType();
968 Type *RetTy = CI->getType();
969
970 ArgListTy Args;
971 Args.reserve(NumArgs);
972
973 // Populate the argument list.
974 // Attributes for args start at offset 1, after the return attribute.
975 for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
976 Value *V = CI->getOperand(ArgI);
977
978 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
979
980 ArgListEntry Entry;
981 Entry.Val = V;
982 Entry.Ty = V->getType();
983 Entry.setAttributes(CI, ArgI);
984 Args.push_back(Entry);
985 }
987
989 CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), *CI, NumArgs);
990
991 return lowerCallTo(CLI);
992}
993
995 // Handle the incoming return values from the call.
996 CLI.clearIns();
997 SmallVector<EVT, 4> RetTys;
998 ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
999
1001 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1002
1003 bool CanLowerReturn = TLI.CanLowerReturn(
1004 CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1005
1006 // FIXME: sret demotion isn't supported yet - bail out.
1007 if (!CanLowerReturn)
1008 return false;
1009
1010 for (EVT VT : RetTys) {
1011 MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1012 unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1013 for (unsigned i = 0; i != NumRegs; ++i) {
1014 ISD::InputArg MyFlags;
1015 MyFlags.VT = RegisterVT;
1016 MyFlags.ArgVT = VT;
1017 MyFlags.Used = CLI.IsReturnValueUsed;
1018 if (CLI.RetSExt)
1019 MyFlags.Flags.setSExt();
1020 if (CLI.RetZExt)
1021 MyFlags.Flags.setZExt();
1022 if (CLI.IsInReg)
1023 MyFlags.Flags.setInReg();
1024 CLI.Ins.push_back(MyFlags);
1025 }
1026 }
1027
1028 // Handle all of the outgoing arguments.
1029 CLI.clearOuts();
1030 for (auto &Arg : CLI.getArgs()) {
1031 Type *FinalType = Arg.Ty;
1032 if (Arg.IsByVal)
1033 FinalType = Arg.IndirectType;
1035 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
1036
1037 ISD::ArgFlagsTy Flags;
1038 if (Arg.IsZExt)
1039 Flags.setZExt();
1040 if (Arg.IsSExt)
1041 Flags.setSExt();
1042 if (Arg.IsInReg)
1043 Flags.setInReg();
1044 if (Arg.IsSRet)
1045 Flags.setSRet();
1046 if (Arg.IsSwiftSelf)
1047 Flags.setSwiftSelf();
1048 if (Arg.IsSwiftAsync)
1049 Flags.setSwiftAsync();
1050 if (Arg.IsSwiftError)
1051 Flags.setSwiftError();
1052 if (Arg.IsCFGuardTarget)
1053 Flags.setCFGuardTarget();
1054 if (Arg.IsByVal)
1055 Flags.setByVal();
1056 if (Arg.IsInAlloca) {
1057 Flags.setInAlloca();
1058 // Set the byval flag for CCAssignFn callbacks that don't know about
1059 // inalloca. This way we can know how many bytes we should've allocated
1060 // and how many bytes a callee cleanup function will pop. If we port
1061 // inalloca to more targets, we'll have to add custom inalloca handling in
1062 // the various CC lowering callbacks.
1063 Flags.setByVal();
1064 }
1065 if (Arg.IsPreallocated) {
1066 Flags.setPreallocated();
1067 // Set the byval flag for CCAssignFn callbacks that don't know about
1068 // preallocated. This way we can know how many bytes we should've
1069 // allocated and how many bytes a callee cleanup function will pop. If we
1070 // port preallocated to more targets, we'll have to add custom
1071 // preallocated handling in the various CC lowering callbacks.
1072 Flags.setByVal();
1073 }
1074 MaybeAlign MemAlign = Arg.Alignment;
1075 if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
1076 unsigned FrameSize = DL.getTypeAllocSize(Arg.IndirectType);
1077
1078 // For ByVal, alignment should come from FE. BE will guess if this info
1079 // is not there, but there are cases it cannot get right.
1080 if (!MemAlign)
1081 MemAlign = Align(TLI.getByValTypeAlignment(Arg.IndirectType, DL));
1082 Flags.setByValSize(FrameSize);
1083 } else if (!MemAlign) {
1084 MemAlign = DL.getABITypeAlign(Arg.Ty);
1085 }
1086 Flags.setMemAlign(*MemAlign);
1087 if (Arg.IsNest)
1088 Flags.setNest();
1089 if (NeedsRegBlock)
1090 Flags.setInConsecutiveRegs();
1091 Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
1092 CLI.OutVals.push_back(Arg.Val);
1093 CLI.OutFlags.push_back(Flags);
1094 }
1095
1096 if (!fastLowerCall(CLI))
1097 return false;
1098
1099 // Set all unused physreg defs as dead.
1100 assert(CLI.Call && "No call instruction specified.");
1102
1103 if (CLI.NumResultRegs && CLI.CB)
1105
1106 // Set labels for heapallocsite call.
1107 if (CLI.CB)
1108 if (MDNode *MD = CLI.CB->getMetadata("heapallocsite"))
1109 CLI.Call->setHeapAllocMarker(*MF, MD);
1110
1111 return true;
1112}
1113
1115 FunctionType *FuncTy = CI->getFunctionType();
1116 Type *RetTy = CI->getType();
1117
1118 ArgListTy Args;
1119 ArgListEntry Entry;
1120 Args.reserve(CI->arg_size());
1121
1122 for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
1123 Value *V = *i;
1124
1125 // Skip empty types
1126 if (V->getType()->isEmptyTy())
1127 continue;
1128
1129 Entry.Val = V;
1130 Entry.Ty = V->getType();
1131
1132 // Skip the first return-type Attribute to get to params.
1133 Entry.setAttributes(CI, i - CI->arg_begin());
1134 Args.push_back(Entry);
1135 }
1136
1137 // Check if target-independent constraints permit a tail call here.
1138 // Target-dependent constraints are checked within fastLowerCall.
1139 bool IsTailCall = CI->isTailCall();
1140 if (IsTailCall && !isInTailCallPosition(*CI, TM))
1141 IsTailCall = false;
1142 if (IsTailCall && !CI->isMustTailCall() &&
1143 MF->getFunction().getFnAttribute("disable-tail-calls").getValueAsBool())
1144 IsTailCall = false;
1145
1146 CallLoweringInfo CLI;
1147 CLI.setCallee(RetTy, FuncTy, CI->getCalledOperand(), std::move(Args), *CI)
1148 .setTailCall(IsTailCall);
1149
1150 diagnoseDontCall(*CI);
1151
1152 return lowerCallTo(CLI);
1153}
1154
1156 const CallInst *Call = cast<CallInst>(I);
1157
1158 // Handle simple inline asms.
1159 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledOperand())) {
1160 // Don't attempt to handle constraints.
1161 if (!IA->getConstraintString().empty())
1162 return false;
1163
1164 unsigned ExtraInfo = 0;
1165 if (IA->hasSideEffects())
1167 if (IA->isAlignStack())
1168 ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1169 if (Call->isConvergent())
1170 ExtraInfo |= InlineAsm::Extra_IsConvergent;
1171 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1172
1174 TII.get(TargetOpcode::INLINEASM));
1175 MIB.addExternalSymbol(IA->getAsmString().c_str());
1176 MIB.addImm(ExtraInfo);
1177
1178 const MDNode *SrcLoc = Call->getMetadata("srcloc");
1179 if (SrcLoc)
1180 MIB.addMetadata(SrcLoc);
1181
1182 return true;
1183 }
1184
1185 // Handle intrinsic function calls.
1186 if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1187 return selectIntrinsicCall(II);
1188
1189 return lowerCall(Call);
1190}
1191
1193 if (!II->hasDbgRecords())
1194 return;
1195
1196 // Clear any metadata.
1197 MIMD = MIMetadata();
1198
1199 // Reverse order of debug records, because fast-isel walks through backwards.
1200 for (DbgRecord &DR : llvm::reverse(II->getDbgRecordRange())) {
1201 flushLocalValueMap();
1203
1204 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1205 assert(DLR->getLabel() && "Missing label");
1206 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1207 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DLR << "\n");
1208 continue;
1209 }
1210
1211 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DLR->getDebugLoc(),
1212 TII.get(TargetOpcode::DBG_LABEL))
1213 .addMetadata(DLR->getLabel());
1214 continue;
1215 }
1216
1217 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
1218
1219 Value *V = nullptr;
1220 if (!DVR.hasArgList())
1221 V = DVR.getVariableLocationOp(0);
1222
1223 bool Res = false;
1226 Res = lowerDbgValue(V, DVR.getExpression(), DVR.getVariable(),
1227 DVR.getDebugLoc());
1228 } else {
1230 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1231 continue;
1232 Res = lowerDbgDeclare(V, DVR.getExpression(), DVR.getVariable(),
1233 DVR.getDebugLoc());
1234 }
1235
1236 if (!Res)
1237 LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DVR << "\n";);
1238 }
1239}
1240
1242 DILocalVariable *Var, const DebugLoc &DL) {
1243 // This form of DBG_VALUE is target-independent.
1244 const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1245 if (!V || isa<UndefValue>(V)) {
1246 // DI is either undef or cannot produce a valid DBG_VALUE, so produce an
1247 // undef DBG_VALUE to terminate any prior location.
1248 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, false, 0U, Var, Expr);
1249 return true;
1250 }
1251 if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1252 // See if there's an expression to constant-fold.
1253 if (Expr)
1254 std::tie(Expr, CI) = Expr->constantFold(CI);
1255 if (CI->getBitWidth() > 64)
1257 .addCImm(CI)
1258 .addImm(0U)
1259 .addMetadata(Var)
1260 .addMetadata(Expr);
1261 else
1263 .addImm(CI->getZExtValue())
1264 .addImm(0U)
1265 .addMetadata(Var)
1266 .addMetadata(Expr);
1267 return true;
1268 }
1269 if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1271 .addFPImm(CF)
1272 .addImm(0U)
1273 .addMetadata(Var)
1274 .addMetadata(Expr);
1275 return true;
1276 }
1277 if (const auto *Arg = dyn_cast<Argument>(V);
1278 Arg && Expr && Expr->isEntryValue()) {
1279 // As per the Verifier, this case is only valid for swift async Args.
1280 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
1281
1282 Register Reg = getRegForValue(Arg);
1283 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
1284 if (Reg == VirtReg || Reg == PhysReg) {
1285 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, false /*IsIndirect*/,
1286 PhysReg, Var, Expr);
1287 return true;
1288 }
1289
1290 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
1291 "couldn't find a physical register\n");
1292 return false;
1293 }
1294 if (auto SI = FuncInfo.StaticAllocaMap.find(dyn_cast<AllocaInst>(V));
1295 SI != FuncInfo.StaticAllocaMap.end()) {
1296 MachineOperand FrameIndexOp = MachineOperand::CreateFI(SI->second);
1297 bool IsIndirect = false;
1298 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, IsIndirect, FrameIndexOp,
1299 Var, Expr);
1300 return true;
1301 }
1302 if (Register Reg = lookUpRegForValue(V)) {
1303 // FIXME: This does not handle register-indirect values at offset 0.
1304 if (!FuncInfo.MF->useDebugInstrRef()) {
1305 bool IsIndirect = false;
1306 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, IsIndirect, Reg, Var,
1307 Expr);
1308 return true;
1309 }
1310 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1311 // to be later patched up by finalizeDebugInstrRefs.
1313 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
1314 /* isKill */ false, /* isDead */ false,
1315 /* isUndef */ false, /* isEarlyClobber */ false,
1316 /* SubReg */ 0, /* isDebug */ true)});
1318 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1320 TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs,
1321 Var, NewExpr);
1322 return true;
1323 }
1324 return false;
1325}
1326
1328 DILocalVariable *Var, const DebugLoc &DL) {
1329 if (!Address || isa<UndefValue>(Address)) {
1330 LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n");
1331 return false;
1332 }
1333
1334 std::optional<MachineOperand> Op;
1336 Op = MachineOperand::CreateReg(Reg, false);
1337
1338 // If we have a VLA that has a "use" in a metadata node that's then used
1339 // here but it has no other uses, then we have a problem. E.g.,
1340 //
1341 // int foo (const int *x) {
1342 // char a[*x];
1343 // return 0;
1344 // }
1345 //
1346 // If we assign 'a' a vreg and fast isel later on has to use the selection
1347 // DAG isel, it will want to copy the value to the vreg. However, there are
1348 // no uses, which goes counter to what selection DAG isel expects.
1349 if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1350 (!isa<AllocaInst>(Address) ||
1351 !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1353 false);
1354
1355 if (Op) {
1357 "Expected inlined-at fields to agree");
1358 if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
1359 // If using instruction referencing, produce this as a DBG_INSTR_REF,
1360 // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
1361 // the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
1363 {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref});
1364 auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1366 TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, *Op,
1367 Var, NewExpr);
1368 return true;
1369 }
1370
1371 // A dbg.declare describes the address of a source variable, so lower it
1372 // into an indirect DBG_VALUE.
1374 TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, *Op, Var,
1375 Expr);
1376 return true;
1377 }
1378
1379 // We can't yet handle anything else here because it would require
1380 // generating code, thus altering codegen because of debug info.
1381 LLVM_DEBUG(
1382 dbgs() << "Dropping debug info (no materialized reg for address)\n");
1383 return false;
1384}
1385
1387 switch (II->getIntrinsicID()) {
1388 default:
1389 break;
1390 // At -O0 we don't care about the lifetime intrinsics.
1391 case Intrinsic::lifetime_start:
1392 case Intrinsic::lifetime_end:
1393 // The donothing intrinsic does, well, nothing.
1394 case Intrinsic::donothing:
1395 // Neither does the sideeffect intrinsic.
1396 case Intrinsic::sideeffect:
1397 // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1398 case Intrinsic::assume:
1399 // Neither does the llvm.experimental.noalias.scope.decl intrinsic
1400 case Intrinsic::experimental_noalias_scope_decl:
1401 return true;
1402 case Intrinsic::dbg_declare: {
1403 const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1404 assert(DI->getVariable() && "Missing variable");
1405 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1406 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI
1407 << " (!hasDebugInfo)\n");
1408 return true;
1409 }
1410
1411 if (FuncInfo.PreprocessedDbgDeclares.contains(DI))
1412 return true;
1413
1414 const Value *Address = DI->getAddress();
1416 MIMD.getDL()))
1417 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI);
1418
1419 return true;
1420 }
1421 case Intrinsic::dbg_assign:
1422 // A dbg.assign is a dbg.value with more information, typically produced
1423 // during optimisation. If one reaches fastisel then something odd has
1424 // happened (such as an optimised function being always-inlined into an
1425 // optnone function). We will not be using the extra information in the
1426 // dbg.assign in that case, just use its dbg.value fields.
1427 [[fallthrough]];
1428 case Intrinsic::dbg_value: {
1429 // This form of DBG_VALUE is target-independent.
1430 const DbgValueInst *DI = cast<DbgValueInst>(II);
1431 const Value *V = DI->getValue();
1432 DIExpression *Expr = DI->getExpression();
1433 DILocalVariable *Var = DI->getVariable();
1434 if (DI->hasArgList())
1435 // Signal that we don't have a location for this.
1436 V = nullptr;
1437
1439 "Expected inlined-at fields to agree");
1440
1441 if (!lowerDbgValue(V, Expr, Var, MIMD.getDL()))
1442 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1443
1444 return true;
1445 }
1446 case Intrinsic::dbg_label: {
1447 const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1448 assert(DI->getLabel() && "Missing label");
1449 if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1450 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1451 return true;
1452 }
1453
1455 TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1456 return true;
1457 }
1458 case Intrinsic::objectsize:
1459 llvm_unreachable("llvm.objectsize.* should have been lowered already");
1460
1461 case Intrinsic::is_constant:
1462 llvm_unreachable("llvm.is.constant.* should have been lowered already");
1463
1464 case Intrinsic::allow_runtime_check:
1465 case Intrinsic::allow_ubsan_check: {
1466 Register ResultReg = getRegForValue(ConstantInt::getTrue(II->getType()));
1467 if (!ResultReg)
1468 return false;
1469 updateValueMap(II, ResultReg);
1470 return true;
1471 }
1472
1473 case Intrinsic::launder_invariant_group:
1474 case Intrinsic::strip_invariant_group:
1475 case Intrinsic::expect: {
1476 Register ResultReg = getRegForValue(II->getArgOperand(0));
1477 if (!ResultReg)
1478 return false;
1479 updateValueMap(II, ResultReg);
1480 return true;
1481 }
1482 case Intrinsic::experimental_stackmap:
1483 return selectStackmap(II);
1484 case Intrinsic::experimental_patchpoint_void:
1485 case Intrinsic::experimental_patchpoint:
1486 return selectPatchpoint(II);
1487
1488 case Intrinsic::xray_customevent:
1489 return selectXRayCustomEvent(II);
1490 case Intrinsic::xray_typedevent:
1491 return selectXRayTypedEvent(II);
1492 }
1493
1494 return fastLowerIntrinsicCall(II);
1495}
1496
1497bool FastISel::selectCast(const User *I, unsigned Opcode) {
1498 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1499 EVT DstVT = TLI.getValueType(DL, I->getType());
1500
1501 if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1502 !DstVT.isSimple())
1503 // Unhandled type. Halt "fast" selection and bail.
1504 return false;
1505
1506 // Check if the destination type is legal.
1507 if (!TLI.isTypeLegal(DstVT))
1508 return false;
1509
1510 // Check if the source operand is legal.
1511 if (!TLI.isTypeLegal(SrcVT))
1512 return false;
1513
1514 Register InputReg = getRegForValue(I->getOperand(0));
1515 if (!InputReg)
1516 // Unhandled operand. Halt "fast" selection and bail.
1517 return false;
1518
1519 Register ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1520 Opcode, InputReg);
1521 if (!ResultReg)
1522 return false;
1523
1524 updateValueMap(I, ResultReg);
1525 return true;
1526}
1527
1529 EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1530 EVT DstEVT = TLI.getValueType(DL, I->getType());
1531 if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1532 !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1533 // Unhandled type. Halt "fast" selection and bail.
1534 return false;
1535
1536 MVT SrcVT = SrcEVT.getSimpleVT();
1537 MVT DstVT = DstEVT.getSimpleVT();
1538 Register Op0 = getRegForValue(I->getOperand(0));
1539 if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1540 return false;
1541
1542 // If the bitcast doesn't change the type, just use the operand value.
1543 if (SrcVT == DstVT) {
1544 updateValueMap(I, Op0);
1545 return true;
1546 }
1547
1548 // Otherwise, select a BITCAST opcode.
1549 Register ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0);
1550 if (!ResultReg)
1551 return false;
1552
1553 updateValueMap(I, ResultReg);
1554 return true;
1555}
1556
1558 Register Reg = getRegForValue(I->getOperand(0));
1559 if (!Reg)
1560 // Unhandled operand.
1561 return false;
1562
1563 EVT ETy = TLI.getValueType(DL, I->getOperand(0)->getType());
1564 if (ETy == MVT::Other || !TLI.isTypeLegal(ETy))
1565 // Unhandled type, bail out.
1566 return false;
1567
1568 MVT Ty = ETy.getSimpleVT();
1569 const TargetRegisterClass *TyRegClass = TLI.getRegClassFor(Ty);
1570 Register ResultReg = createResultReg(TyRegClass);
1572 TII.get(TargetOpcode::COPY), ResultReg).addReg(Reg);
1573
1574 updateValueMap(I, ResultReg);
1575 return true;
1576}
1577
1578// Remove local value instructions starting from the instruction after
1579// SavedLastLocalValue to the current function insert point.
1580void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1581{
1582 MachineInstr *CurLastLocalValue = getLastLocalValue();
1583 if (CurLastLocalValue != SavedLastLocalValue) {
1584 // Find the first local value instruction to be deleted.
1585 // This is the instruction after SavedLastLocalValue if it is non-NULL.
1586 // Otherwise it's the first instruction in the block.
1587 MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1588 if (SavedLastLocalValue)
1589 ++FirstDeadInst;
1590 else
1591 FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1592 setLastLocalValue(SavedLastLocalValue);
1593 removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1594 }
1595}
1596
1598 // Flush the local value map before starting each instruction.
1599 // This improves locality and debugging, and can reduce spills.
1600 // Reuse of values across IR instructions is relatively uncommon.
1601 flushLocalValueMap();
1602
1603 MachineInstr *SavedLastLocalValue = getLastLocalValue();
1604 // Just before the terminator instruction, insert instructions to
1605 // feed PHI nodes in successor blocks.
1606 if (I->isTerminator()) {
1607 if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1608 // PHI node handling may have generated local value instructions,
1609 // even though it failed to handle all PHI nodes.
1610 // We remove these instructions because SelectionDAGISel will generate
1611 // them again.
1612 removeDeadLocalValueCode(SavedLastLocalValue);
1613 return false;
1614 }
1615 }
1616
1617 // FastISel does not handle any operand bundles except OB_funclet.
1618 if (auto *Call = dyn_cast<CallBase>(I))
1619 for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i)
1620 if (Call->getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1621 return false;
1622
1623 MIMD = MIMetadata(*I);
1624
1625 SavedInsertPt = FuncInfo.InsertPt;
1626
1627 if (const auto *Call = dyn_cast<CallInst>(I)) {
1628 const Function *F = Call->getCalledFunction();
1629 LibFunc Func;
1630
1631 // As a special case, don't handle calls to builtin library functions that
1632 // may be translated directly to target instructions.
1633 if (F && !F->hasLocalLinkage() && F->hasName() &&
1634 LibInfo->getLibFunc(F->getName(), Func) &&
1636 return false;
1637
1638 // Don't handle Intrinsic::trap if a trap function is specified.
1639 if (F && F->getIntrinsicID() == Intrinsic::trap &&
1640 Call->hasFnAttr("trap-func-name"))
1641 return false;
1642 }
1643
1644 // First, try doing target-independent selection.
1646 if (selectOperator(I, I->getOpcode())) {
1647 ++NumFastIselSuccessIndependent;
1648 MIMD = {};
1649 return true;
1650 }
1651 // Remove dead code.
1653 if (SavedInsertPt != FuncInfo.InsertPt)
1654 removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1655 SavedInsertPt = FuncInfo.InsertPt;
1656 }
1657 // Next, try calling the target to attempt to handle the instruction.
1658 if (fastSelectInstruction(I)) {
1659 ++NumFastIselSuccessTarget;
1660 MIMD = {};
1661 return true;
1662 }
1663 // Remove dead code.
1665 if (SavedInsertPt != FuncInfo.InsertPt)
1666 removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1667
1668 MIMD = {};
1669 // Undo phi node updates, because they will be added again by SelectionDAG.
1670 if (I->isTerminator()) {
1671 // PHI node handling may have generated local value instructions.
1672 // We remove them because SelectionDAGISel will generate them again.
1673 removeDeadLocalValueCode(SavedLastLocalValue);
1675 }
1676 return false;
1677}
1678
1679/// Emit an unconditional branch to the given block, unless it is the immediate
1680/// (fall-through) successor, and update the CFG.
1682 const DebugLoc &DbgLoc) {
1684 FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1685 // For more accurate line information if this is the only non-debug
1686 // instruction in the block then emit it, otherwise we have the
1687 // unconditional fall-through case, which needs no instructions.
1688 } else {
1689 // The unconditional branch case.
1690 TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1692 }
1693 if (FuncInfo.BPI) {
1697 } else
1699}
1700
1702 MachineBasicBlock *TrueMBB,
1703 MachineBasicBlock *FalseMBB) {
1704 // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1705 // happen in degenerate IR and MachineIR forbids to have a block twice in the
1706 // successor/predecessor lists.
1707 if (TrueMBB != FalseMBB) {
1708 if (FuncInfo.BPI) {
1709 auto BranchProbability =
1710 FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1712 } else
1714 }
1715
1716 fastEmitBranch(FalseMBB, MIMD.getDL());
1717}
1718
1719/// Emit an FNeg operation.
1720bool FastISel::selectFNeg(const User *I, const Value *In) {
1721 Register OpReg = getRegForValue(In);
1722 if (!OpReg)
1723 return false;
1724
1725 // If the target has ISD::FNEG, use it.
1726 EVT VT = TLI.getValueType(DL, I->getType());
1727 Register ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1728 OpReg);
1729 if (ResultReg) {
1730 updateValueMap(I, ResultReg);
1731 return true;
1732 }
1733
1734 // Bitcast the value to integer, twiddle the sign bit with xor,
1735 // and then bitcast it back to floating-point.
1736 if (VT.getSizeInBits() > 64)
1737 return false;
1738 EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1739 if (!TLI.isTypeLegal(IntVT))
1740 return false;
1741
1742 Register IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1743 ISD::BITCAST, OpReg);
1744 if (!IntReg)
1745 return false;
1746
1747 Register IntResultReg = fastEmit_ri_(
1748 IntVT.getSimpleVT(), ISD::XOR, IntReg,
1749 UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1750 if (!IntResultReg)
1751 return false;
1752
1753 ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1754 IntResultReg);
1755 if (!ResultReg)
1756 return false;
1757
1758 updateValueMap(I, ResultReg);
1759 return true;
1760}
1761
1763 const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1764 if (!EVI)
1765 return false;
1766
1767 // Make sure we only try to handle extracts with a legal result. But also
1768 // allow i1 because it's easy.
1769 EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1770 if (!RealVT.isSimple())
1771 return false;
1772 MVT VT = RealVT.getSimpleVT();
1773 if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1774 return false;
1775
1776 const Value *Op0 = EVI->getOperand(0);
1777 Type *AggTy = Op0->getType();
1778
1779 // Get the base result register.
1780 unsigned ResultReg;
1782 if (I != FuncInfo.ValueMap.end())
1783 ResultReg = I->second;
1784 else if (isa<Instruction>(Op0))
1785 ResultReg = FuncInfo.InitializeRegForValue(Op0);
1786 else
1787 return false; // fast-isel can't handle aggregate constants at the moment
1788
1789 // Get the actual result register, which is an offset from the base register.
1790 unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1791
1792 SmallVector<EVT, 4> AggValueVTs;
1793 ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1794
1795 for (unsigned i = 0; i < VTIndex; i++)
1796 ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1797
1798 updateValueMap(EVI, ResultReg);
1799 return true;
1800}
1801
1802bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1803 switch (Opcode) {
1804 case Instruction::Add:
1805 return selectBinaryOp(I, ISD::ADD);
1806 case Instruction::FAdd:
1807 return selectBinaryOp(I, ISD::FADD);
1808 case Instruction::Sub:
1809 return selectBinaryOp(I, ISD::SUB);
1810 case Instruction::FSub:
1811 return selectBinaryOp(I, ISD::FSUB);
1812 case Instruction::Mul:
1813 return selectBinaryOp(I, ISD::MUL);
1814 case Instruction::FMul:
1815 return selectBinaryOp(I, ISD::FMUL);
1816 case Instruction::SDiv:
1817 return selectBinaryOp(I, ISD::SDIV);
1818 case Instruction::UDiv:
1819 return selectBinaryOp(I, ISD::UDIV);
1820 case Instruction::FDiv:
1821 return selectBinaryOp(I, ISD::FDIV);
1822 case Instruction::SRem:
1823 return selectBinaryOp(I, ISD::SREM);
1824 case Instruction::URem:
1825 return selectBinaryOp(I, ISD::UREM);
1826 case Instruction::FRem:
1827 return selectBinaryOp(I, ISD::FREM);
1828 case Instruction::Shl:
1829 return selectBinaryOp(I, ISD::SHL);
1830 case Instruction::LShr:
1831 return selectBinaryOp(I, ISD::SRL);
1832 case Instruction::AShr:
1833 return selectBinaryOp(I, ISD::SRA);
1834 case Instruction::And:
1835 return selectBinaryOp(I, ISD::AND);
1836 case Instruction::Or:
1837 return selectBinaryOp(I, ISD::OR);
1838 case Instruction::Xor:
1839 return selectBinaryOp(I, ISD::XOR);
1840
1841 case Instruction::FNeg:
1842 return selectFNeg(I, I->getOperand(0));
1843
1844 case Instruction::GetElementPtr:
1845 return selectGetElementPtr(I);
1846
1847 case Instruction::Br: {
1848 const BranchInst *BI = cast<BranchInst>(I);
1849
1850 if (BI->isUnconditional()) {
1851 const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1852 MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1853 fastEmitBranch(MSucc, BI->getDebugLoc());
1854 return true;
1855 }
1856
1857 // Conditional branches are not handed yet.
1858 // Halt "fast" selection and bail.
1859 return false;
1860 }
1861
1862 case Instruction::Unreachable:
1864 return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1865 else
1866 return true;
1867
1868 case Instruction::Alloca:
1869 // FunctionLowering has the static-sized case covered.
1870 if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1871 return true;
1872
1873 // Dynamic-sized alloca is not handled yet.
1874 return false;
1875
1876 case Instruction::Call:
1877 // On AIX, normal call lowering uses the DAG-ISEL path currently so that the
1878 // callee of the direct function call instruction will be mapped to the
1879 // symbol for the function's entry point, which is distinct from the
1880 // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1881 // name is the C-linkage name of the source level function.
1882 // But fast isel still has the ability to do selection for intrinsics.
1883 if (TM.getTargetTriple().isOSAIX() && !isa<IntrinsicInst>(I))
1884 return false;
1885 return selectCall(I);
1886
1887 case Instruction::BitCast:
1888 return selectBitCast(I);
1889
1890 case Instruction::FPToSI:
1891 return selectCast(I, ISD::FP_TO_SINT);
1892 case Instruction::ZExt:
1893 return selectCast(I, ISD::ZERO_EXTEND);
1894 case Instruction::SExt:
1895 return selectCast(I, ISD::SIGN_EXTEND);
1896 case Instruction::Trunc:
1897 return selectCast(I, ISD::TRUNCATE);
1898 case Instruction::SIToFP:
1899 return selectCast(I, ISD::SINT_TO_FP);
1900
1901 case Instruction::IntToPtr: // Deliberate fall-through.
1902 case Instruction::PtrToInt: {
1903 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1904 EVT DstVT = TLI.getValueType(DL, I->getType());
1905 if (DstVT.bitsGT(SrcVT))
1906 return selectCast(I, ISD::ZERO_EXTEND);
1907 if (DstVT.bitsLT(SrcVT))
1908 return selectCast(I, ISD::TRUNCATE);
1909 Register Reg = getRegForValue(I->getOperand(0));
1910 if (!Reg)
1911 return false;
1912 updateValueMap(I, Reg);
1913 return true;
1914 }
1915
1916 case Instruction::ExtractValue:
1917 return selectExtractValue(I);
1918
1919 case Instruction::Freeze:
1920 return selectFreeze(I);
1921
1922 case Instruction::PHI:
1923 llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1924
1925 default:
1926 // Unhandled instruction. Halt "fast" selection and bail.
1927 return false;
1928 }
1929}
1930
1934 : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1935 MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1936 TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1937 TII(*MF->getSubtarget().getInstrInfo()),
1938 TLI(*MF->getSubtarget().getTargetLowering()),
1939 TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1941
1942FastISel::~FastISel() = default;
1943
1944bool FastISel::fastLowerArguments() { return false; }
1945
1946bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1947
1949 return false;
1950}
1951
1952unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1953
1954unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/) {
1955 return 0;
1956}
1957
1958unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1959 unsigned /*Op1*/) {
1960 return 0;
1961}
1962
1963unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1964 return 0;
1965}
1966
1967unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1968 const ConstantFP * /*FPImm*/) {
1969 return 0;
1970}
1971
1972unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1973 uint64_t /*Imm*/) {
1974 return 0;
1975}
1976
1977/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1978/// instruction with an immediate operand using fastEmit_ri.
1979/// If that fails, it materializes the immediate into a register and try
1980/// fastEmit_rr instead.
1981Register FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1982 uint64_t Imm, MVT ImmType) {
1983 // If this is a multiply by a power of two, emit this as a shift left.
1984 if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1985 Opcode = ISD::SHL;
1986 Imm = Log2_64(Imm);
1987 } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1988 // div x, 8 -> srl x, 3
1989 Opcode = ISD::SRL;
1990 Imm = Log2_64(Imm);
1991 }
1992
1993 // Horrible hack (to be removed), check to make sure shift amounts are
1994 // in-range.
1995 if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1996 Imm >= VT.getSizeInBits())
1997 return 0;
1998
1999 // First check if immediate type is legal. If not, we can't use the ri form.
2000 Register ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Imm);
2001 if (ResultReg)
2002 return ResultReg;
2003 Register MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
2004 if (!MaterialReg) {
2005 // This is a bit ugly/slow, but failing here means falling out of
2006 // fast-isel, which would be very slow.
2007 IntegerType *ITy =
2009 MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
2010 if (!MaterialReg)
2011 return 0;
2012 }
2013 return fastEmit_rr(VT, VT, Opcode, Op0, MaterialReg);
2014}
2015
2017 return MRI.createVirtualRegister(RC);
2018}
2019
2021 unsigned OpNum) {
2022 if (Op.isVirtual()) {
2023 const TargetRegisterClass *RegClass =
2024 TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
2025 if (!MRI.constrainRegClass(Op, RegClass)) {
2026 // If it's not legal to COPY between the register classes, something
2027 // has gone very wrong before we got here.
2028 Register NewOp = createResultReg(RegClass);
2030 TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2031 return NewOp;
2032 }
2033 }
2034 return Op;
2035}
2036
2037Register FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2038 const TargetRegisterClass *RC) {
2039 Register ResultReg = createResultReg(RC);
2040 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2041
2042 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg);
2043 return ResultReg;
2044}
2045
2046Register FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2047 const TargetRegisterClass *RC, unsigned Op0) {
2048 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2049
2050 Register ResultReg = createResultReg(RC);
2051 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2052
2053 if (II.getNumDefs() >= 1)
2054 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2055 .addReg(Op0);
2056 else {
2058 .addReg(Op0);
2059 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2060 ResultReg)
2061 .addReg(II.implicit_defs()[0]);
2062 }
2063
2064 return ResultReg;
2065}
2066
2067Register FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2068 const TargetRegisterClass *RC, unsigned Op0,
2069 unsigned Op1) {
2070 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2071
2072 Register ResultReg = createResultReg(RC);
2073 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2074 Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2075
2076 if (II.getNumDefs() >= 1)
2077 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2078 .addReg(Op0)
2079 .addReg(Op1);
2080 else {
2082 .addReg(Op0)
2083 .addReg(Op1);
2084 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2085 ResultReg)
2086 .addReg(II.implicit_defs()[0]);
2087 }
2088 return ResultReg;
2089}
2090
2091Register FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2092 const TargetRegisterClass *RC, unsigned Op0,
2093 unsigned Op1, unsigned Op2) {
2094 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2095
2096 Register ResultReg = createResultReg(RC);
2097 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2098 Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2099 Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2100
2101 if (II.getNumDefs() >= 1)
2102 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2103 .addReg(Op0)
2104 .addReg(Op1)
2105 .addReg(Op2);
2106 else {
2108 .addReg(Op0)
2109 .addReg(Op1)
2110 .addReg(Op2);
2111 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2112 ResultReg)
2113 .addReg(II.implicit_defs()[0]);
2114 }
2115 return ResultReg;
2116}
2117
2118Register FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2119 const TargetRegisterClass *RC, unsigned Op0,
2120 uint64_t Imm) {
2121 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2122
2123 Register ResultReg = createResultReg(RC);
2124 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2125
2126 if (II.getNumDefs() >= 1)
2127 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2128 .addReg(Op0)
2129 .addImm(Imm);
2130 else {
2132 .addReg(Op0)
2133 .addImm(Imm);
2134 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2135 ResultReg)
2136 .addReg(II.implicit_defs()[0]);
2137 }
2138 return ResultReg;
2139}
2140
2141Register FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2142 const TargetRegisterClass *RC, unsigned Op0,
2143 uint64_t Imm1, uint64_t Imm2) {
2144 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2145
2146 Register ResultReg = createResultReg(RC);
2147 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2148
2149 if (II.getNumDefs() >= 1)
2150 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2151 .addReg(Op0)
2152 .addImm(Imm1)
2153 .addImm(Imm2);
2154 else {
2156 .addReg(Op0)
2157 .addImm(Imm1)
2158 .addImm(Imm2);
2159 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2160 ResultReg)
2161 .addReg(II.implicit_defs()[0]);
2162 }
2163 return ResultReg;
2164}
2165
2166Register FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2167 const TargetRegisterClass *RC,
2168 const ConstantFP *FPImm) {
2169 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2170
2171 Register ResultReg = createResultReg(RC);
2172
2173 if (II.getNumDefs() >= 1)
2174 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2175 .addFPImm(FPImm);
2176 else {
2178 .addFPImm(FPImm);
2179 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2180 ResultReg)
2181 .addReg(II.implicit_defs()[0]);
2182 }
2183 return ResultReg;
2184}
2185
2186Register FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2187 const TargetRegisterClass *RC, unsigned Op0,
2188 unsigned Op1, uint64_t Imm) {
2189 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2190
2191 Register ResultReg = createResultReg(RC);
2192 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2193 Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2194
2195 if (II.getNumDefs() >= 1)
2196 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2197 .addReg(Op0)
2198 .addReg(Op1)
2199 .addImm(Imm);
2200 else {
2202 .addReg(Op0)
2203 .addReg(Op1)
2204 .addImm(Imm);
2205 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2206 ResultReg)
2207 .addReg(II.implicit_defs()[0]);
2208 }
2209 return ResultReg;
2210}
2211
2212Register FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2213 const TargetRegisterClass *RC, uint64_t Imm) {
2214 Register ResultReg = createResultReg(RC);
2215 const MCInstrDesc &II = TII.get(MachineInstOpcode);
2216
2217 if (II.getNumDefs() >= 1)
2218 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
2219 .addImm(Imm);
2220 else {
2222 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2223 ResultReg)
2224 .addReg(II.implicit_defs()[0]);
2225 }
2226 return ResultReg;
2227}
2228
2230 uint32_t Idx) {
2231 Register ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2233 "Cannot yet extract from physregs");
2234 const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2236 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2237 ResultReg).addReg(Op0, 0, Idx);
2238 return ResultReg;
2239}
2240
2241/// Emit MachineInstrs to compute the value of Op with all but the least
2242/// significant bit set to zero.
2244 return fastEmit_ri(VT, VT, ISD::AND, Op0, 1);
2245}
2246
2247/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2248/// Emit code to ensure constants are copied into registers when needed.
2249/// Remember the virtual registers that need to be added to the Machine PHI
2250/// nodes as input. We cannot just directly add them, because expansion
2251/// might result in multiple MBB's for one BB. As such, the start of the
2252/// BB might correspond to a different MBB than the end.
2253bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2256
2257 // Check successor nodes' PHI nodes that expect a constant to be available
2258 // from this block.
2259 for (const BasicBlock *SuccBB : successors(LLVMBB)) {
2260 if (!isa<PHINode>(SuccBB->begin()))
2261 continue;
2262 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2263
2264 // If this terminator has multiple identical successors (common for
2265 // switches), only handle each succ once.
2266 if (!SuccsHandled.insert(SuccMBB).second)
2267 continue;
2268
2270
2271 // At this point we know that there is a 1-1 correspondence between LLVM PHI
2272 // nodes and Machine PHI nodes, but the incoming operands have not been
2273 // emitted yet.
2274 for (const PHINode &PN : SuccBB->phis()) {
2275 // Ignore dead phi's.
2276 if (PN.use_empty())
2277 continue;
2278
2279 // Only handle legal types. Two interesting things to note here. First,
2280 // by bailing out early, we may leave behind some dead instructions,
2281 // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2282 // own moves. Second, this check is necessary because FastISel doesn't
2283 // use CreateRegs to create registers, so it always creates
2284 // exactly one register for each non-void instruction.
2285 EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2286 if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2287 // Handle integer promotions, though, because they're common and easy.
2288 if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2290 return false;
2291 }
2292 }
2293
2294 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2295
2296 // Set the DebugLoc for the copy. Use the location of the operand if
2297 // there is one; otherwise no location, flushLocalValueMap will fix it.
2298 MIMD = {};
2299 if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2300 MIMD = MIMetadata(*Inst);
2301
2302 Register Reg = getRegForValue(PHIOp);
2303 if (!Reg) {
2305 return false;
2306 }
2307 FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2308 MIMD = {};
2309 }
2310 }
2311
2312 return true;
2313}
2314
2315bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2316 assert(LI->hasOneUse() &&
2317 "tryToFoldLoad expected a LoadInst with a single use");
2318 // We know that the load has a single use, but don't know what it is. If it
2319 // isn't one of the folded instructions, then we can't succeed here. Handle
2320 // this by scanning the single-use users of the load until we get to FoldInst.
2321 unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2322
2323 const Instruction *TheUser = LI->user_back();
2324 while (TheUser != FoldInst && // Scan up until we find FoldInst.
2325 // Stay in the right block.
2326 TheUser->getParent() == FoldInst->getParent() &&
2327 --MaxUsers) { // Don't scan too far.
2328 // If there are multiple or no uses of this instruction, then bail out.
2329 if (!TheUser->hasOneUse())
2330 return false;
2331
2332 TheUser = TheUser->user_back();
2333 }
2334
2335 // If we didn't find the fold instruction, then we failed to collapse the
2336 // sequence.
2337 if (TheUser != FoldInst)
2338 return false;
2339
2340 // Don't try to fold volatile loads. Target has to deal with alignment
2341 // constraints.
2342 if (LI->isVolatile())
2343 return false;
2344
2345 // Figure out which vreg this is going into. If there is no assigned vreg yet
2346 // then there actually was no reference to it. Perhaps the load is referenced
2347 // by a dead instruction.
2348 Register LoadReg = getRegForValue(LI);
2349 if (!LoadReg)
2350 return false;
2351
2352 // We can't fold if this vreg has no uses or more than one use. Multiple uses
2353 // may mean that the instruction got lowered to multiple MIs, or the use of
2354 // the loaded value ended up being multiple operands of the result.
2355 if (!MRI.hasOneUse(LoadReg))
2356 return false;
2357
2358 // If the register has fixups, there may be additional uses through a
2359 // different alias of the register.
2360 if (FuncInfo.RegsWithFixups.contains(LoadReg))
2361 return false;
2362
2364 MachineInstr *User = RI->getParent();
2365
2366 // Set the insertion point properly. Folding the load can cause generation of
2367 // other random instructions (like sign extends) for addressing modes; make
2368 // sure they get inserted in a logical place before the new instruction.
2370 FuncInfo.MBB = User->getParent();
2371
2372 // Ask the target to try folding the load.
2373 return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2374}
2375
2377 // Must be an add.
2378 if (!isa<AddOperator>(Add))
2379 return false;
2380 // Type size needs to match.
2381 if (DL.getTypeSizeInBits(GEP->getType()) !=
2382 DL.getTypeSizeInBits(Add->getType()))
2383 return false;
2384 // Must be in the same basic block.
2385 if (isa<Instruction>(Add) &&
2386 FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2387 return false;
2388 // Must have a constant operand.
2389 return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2390}
2391
2394 const Value *Ptr;
2395 Type *ValTy;
2396 MaybeAlign Alignment;
2398 bool IsVolatile;
2399
2400 if (const auto *LI = dyn_cast<LoadInst>(I)) {
2401 Alignment = LI->getAlign();
2402 IsVolatile = LI->isVolatile();
2404 Ptr = LI->getPointerOperand();
2405 ValTy = LI->getType();
2406 } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2407 Alignment = SI->getAlign();
2408 IsVolatile = SI->isVolatile();
2410 Ptr = SI->getPointerOperand();
2411 ValTy = SI->getValueOperand()->getType();
2412 } else
2413 return nullptr;
2414
2415 bool IsNonTemporal = I->hasMetadata(LLVMContext::MD_nontemporal);
2416 bool IsInvariant = I->hasMetadata(LLVMContext::MD_invariant_load);
2417 bool IsDereferenceable = I->hasMetadata(LLVMContext::MD_dereferenceable);
2418 const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2419
2420 AAMDNodes AAInfo = I->getAAMetadata();
2421
2422 if (!Alignment) // Ensure that codegen never sees alignment 0.
2423 Alignment = DL.getABITypeAlign(ValTy);
2424
2425 unsigned Size = DL.getTypeStoreSize(ValTy);
2426
2427 if (IsVolatile)
2429 if (IsNonTemporal)
2431 if (IsDereferenceable)
2433 if (IsInvariant)
2435
2437 *Alignment, AAInfo, Ranges);
2438}
2439
2441 // If both operands are the same, then try to optimize or fold the cmp.
2442 CmpInst::Predicate Predicate = CI->getPredicate();
2443 if (CI->getOperand(0) != CI->getOperand(1))
2444 return Predicate;
2445
2446 switch (Predicate) {
2447 default: llvm_unreachable("Invalid predicate!");
2448 case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2449 case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2450 case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2451 case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2452 case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2453 case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2454 case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2455 case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2456 case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2457 case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2458 case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2459 case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2460 case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2461 case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2462 case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2463 case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2464
2465 case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2466 case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2467 case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2468 case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2469 case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2470 case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2471 case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2472 case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2473 case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2474 case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2475 }
2476
2477 return Predicate;
2478}
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file contains the simple types necessary to represent the attributes associated with functions a...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:479
static Register findLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:160
static bool isRegUsedByPhiNodes(Register DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:177
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:943
This file defines the FastISel class.
Hexagon Common GEP
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file contains the declarations for metadata subclasses.
uint64_t IntrinsicInst * II
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallPtrSet class.
This file defines the SmallString class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This file describes how to lower LLVM code to machine code.
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
bool getValueAsBool() const
Return the attribute's value as a boolean.
Definition: Attributes.cpp:377
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
filter_iterator< BasicBlock::const_iterator, std::function< bool(constInstruction &)> >::difference_type sizeWithoutDebug() const
Return the size of the basic block ignoring debug instructions.
Definition: BasicBlock.cpp:268
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1523
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1385
Value * getCalledOperand() const
Definition: InstrTypes.h:1458
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1391
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1323
unsigned arg_size() const
Definition: InstrTypes.h:1408
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool isMustTailCall() const
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:747
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:757
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:760
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:774
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:786
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:787
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:763
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:772
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:761
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:762
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:781
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:780
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:784
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:771
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:765
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:768
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:782
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:769
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:764
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:766
@ ICMP_EQ
equal
Definition: InstrTypes.h:778
@ ICMP_NE
not equal
Definition: InstrTypes.h:779
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:785
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:773
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:783
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:770
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:759
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:767
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:847
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:269
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:850
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
std::pair< DIExpression *, const ConstantInt * > constantFold(const ConstantInt *CI)
Try to shorten an expression with an initial constant operand.
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:878
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:865
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:504
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:672
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
A debug info location.
Definition: DebugLoc.h:33
This instruction extracts a struct member or array element value from an aggregate value.
ArrayRef< unsigned > getIndices() const
MachineRegisterInfo & MRI
Definition: FastISel.h:205
Register fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2229
const TargetLibraryInfo * LibInfo
Definition: FastISel.h:214
const DataLayout & DL
Definition: FastISel.h:210
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:531
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:237
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:642
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1762
DenseMap< const Value *, Register > LocalValueMap
Definition: FastISel.h:202
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor,...
Definition: FastISel.cpp:1681
virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF)
Emit the floating-point constant +0.0 in a register using target- specific logic.
Definition: FastISel.h:480
MachineInstr * EmitStartPt
The top most instruction in the current block that is allowed for emitting local variables.
Definition: FastISel.h:226
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:901
Register fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2037
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering.
Definition: FastISel.cpp:1948
virtual bool lowerDbgDeclare(const Value *V, DIExpression *Expr, DILocalVariable *Var, const DebugLoc &DL)
Target-independent lowering of debug information.
Definition: FastISel.cpp:1327
MachineInstr * getLastLocalValue()
Return the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:233
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1114
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1972
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:436
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:965
Register fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0)
Emit a MachineInstr with one register operand and a result register in the given register class.
Definition: FastISel.cpp:2046
void handleDbgInfo(const Instruction *II)
Target-independent lowering of non-instruction debug info associated with this instruction.
Definition: FastISel.cpp:1192
bool selectFreeze(const User *I)
Definition: FastISel.cpp:1557
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1386
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1497
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2315
Register getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value.
Definition: FastISel.cpp:238
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:410
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:123
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2393
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:300
Register fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2212
MachineFrameInfo & MFI
Definition: FastISel.h:206
MachineFunction * MF
Definition: FastISel.h:204
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2376
virtual bool lowerDbgValue(const Value *V, DIExpression *Expr, DILocalVariable *Var, const DebugLoc &DL)
Target-independent lowering of debug information.
Definition: FastISel.cpp:1241
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1952
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:69
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1597
virtual unsigned fastMaterializeConstant(const Constant *C)
Emit a constant in a register using target-specific logic, such as constant pool loads.
Definition: FastISel.h:473
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1946
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:920
Register fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, unsigned Op1)
Emit a MachineInstr with two register operands and a result register in the given register class.
Definition: FastISel.cpp:2067
Register fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2141
Register createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:2016
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1967
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1944
bool selectFNeg(const User *I, const Value *In)
Emit an FNeg operation.
Definition: FastISel.cpp:1720
const TargetInstrInfo & TII
Definition: FastISel.h:211
bool selectCall(const User *I)
Definition: FastISel.cpp:1155
Register lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:351
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2440
void finishBasicBlock()
Flush the local value map.
Definition: FastISel.cpp:136
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:203
Register getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:383
MachineConstantPool & MCP
Definition: FastISel.h:207
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr),...
Definition: FastISel.cpp:1802
bool SkipTargetIndependentISel
Definition: FastISel.h:215
Register fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2166
Register constrainOperandRegClass(const MCInstrDesc &II, Register Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:2020
void updateValueMap(const Value *I, Register Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:362
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:444
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1931
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:754
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:401
virtual bool fastSelectInstruction(const Instruction *I)=0
This method is called by target-independent code when the normal FastISel process fails to select an ...
const TargetLowering & TLI
Definition: FastISel.h:212
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1958
const TargetMachine & TM
Definition: FastISel.h:209
Register fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1981
Register fastEmitZExtFromI1(MVT VT, unsigned Op0)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero.
Definition: FastISel.cpp:2243
MIMetadata MIMD
Definition: FastISel.h:208
MachineInstr * LastLocalValue
The position of the last instruction for materializing constants for use in the current block.
Definition: FastISel.h:221
Register fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, unsigned Op1, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2186
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:138
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:430
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1701
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1528
Register fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2118
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1954
virtual unsigned fastMaterializeAlloca(const AllocaInst *C)
Emit an alloca address in a register using target-specific logic.
Definition: FastISel.h:476
virtual ~FastISel()
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type,...
Definition: FastISel.cpp:1963
const TargetRegisterInfo & TRI
Definition: FastISel.h:213
Register fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, unsigned Op1, unsigned Op2)
Emit a MachineInstr with three register operands and a result register in the given register class.
Definition: FastISel.cpp:2091
TargetLoweringBase::ArgListEntry ArgListEntry
Definition: FastISel.h:68
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
DenseSet< Register > RegsWithFixups
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
Register InitializeRegForValue(const Value *V)
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
DenseMap< Register, Register > RegFixups
RegFixups - Registers which need to be replaced after isel is done.
MachineRegisterInfo * RegInfo
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
Class to represent function types.
Definition: DerivedTypes.h:103
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:745
arg_iterator arg_end()
Definition: Function.h:840
arg_iterator arg_begin()
Definition: Function.h:831
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:358
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:476
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:169
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:381
Class to represent integer types.
Definition: DerivedTypes.h:40
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:278
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:48
An instruction for reading from memory.
Definition: Instructions.h:174
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:203
Context object for machine code objects.
Definition: MCContext.h:83
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:213
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:237
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition: MCInstrInfo.h:63
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:1067
Set of metadata that should be preserved when using BuildMI().
const DebugLoc & getDL() const
Machine Value Type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
MachineInstrBundleIterator< MachineInstr > iterator
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
void setHeapAllocMarker(MachineFunction &MF, MDNode *MD)
Set a marker on instructions that denotes where we should create and emit heap alloc site labels.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void setPhysRegsDeadExcept(ArrayRef< Register > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
bool hasDebugInfo() const
Returns true if valid debug info is present.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
reg_iterator reg_begin(Register RegNo) const
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
ArrayRef< std::pair< MCRegister, Register > > liveins() const
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable's name.
Definition: Mangler.cpp:120
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:344
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:479
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const
Insert branch code into the end of the specified MachineBasicBlock.
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise).
unsigned getCallFrameDestroyOpcode() const
virtual const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum,...
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const
Returns a 0 terminated array of registers that can be safely used as scratch registers.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
const Triple & getTargetTriple() const
TargetOptions Options
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:371
bool isOSAIX() const
Tests whether the OS is AIX.
Definition: Triple.h:708
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Definition: Triple.h:909
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:810
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:923
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:794
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:950
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:725
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:800
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:856
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:700
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1241
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:806
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
Definition: ISDOpcodes.h:1458
Reg
All possible values of the reg field in the ModR/M byte.
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:147
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:227
reverse_iterator rend(StringRef path)
Get reverse end iterator over path.
Definition: Path.cpp:307
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:296
gep_type_iterator gep_type_end(const User *GEP)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:346
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
@ Add
Sum of integers.
DWARFExpression::Operation Op
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
gep_type_iterator gep_type_begin(const User *GEP)
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:250
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:274
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:95
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:94
SmallVector< Register, 16 > OutRegs
Definition: FastISel.h:96
CallLoweringInfo & setTailCall(bool Value=true)
Definition: FastISel.h:177
SmallVector< Register, 4 > InRegs
Definition: FastISel.h:98
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:182
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, const CallBase &Call)
Definition: FastISel.h:104
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:97
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
This class contains a discriminated union of information about pointers in memory operands,...
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117