LLVM  10.0.0svn
SIISelLowering.cpp
Go to the documentation of this file.
1 //===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Custom DAG lowering for SI
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #if defined(_MSC_VER) || defined(__MINGW32__)
15 // Provide M_PI.
16 #define _USE_MATH_DEFINES
17 #endif
18 
19 #include "SIISelLowering.h"
20 #include "AMDGPU.h"
21 #include "AMDGPUSubtarget.h"
22 #include "AMDGPUTargetMachine.h"
24 #include "SIDefines.h"
25 #include "SIInstrInfo.h"
26 #include "SIMachineFunctionInfo.h"
27 #include "SIRegisterInfo.h"
28 #include "Utils/AMDGPUBaseInfo.h"
29 #include "llvm/ADT/APFloat.h"
30 #include "llvm/ADT/APInt.h"
31 #include "llvm/ADT/ArrayRef.h"
32 #include "llvm/ADT/BitVector.h"
33 #include "llvm/ADT/SmallVector.h"
34 #include "llvm/ADT/Statistic.h"
35 #include "llvm/ADT/StringRef.h"
36 #include "llvm/ADT/StringSwitch.h"
37 #include "llvm/ADT/Twine.h"
39 #include "llvm/CodeGen/Analysis.h"
58 #include "llvm/IR/Constants.h"
59 #include "llvm/IR/DataLayout.h"
60 #include "llvm/IR/DebugLoc.h"
61 #include "llvm/IR/DerivedTypes.h"
62 #include "llvm/IR/DiagnosticInfo.h"
63 #include "llvm/IR/Function.h"
64 #include "llvm/IR/GlobalValue.h"
65 #include "llvm/IR/InstrTypes.h"
66 #include "llvm/IR/Instruction.h"
67 #include "llvm/IR/Instructions.h"
68 #include "llvm/IR/IntrinsicInst.h"
69 #include "llvm/IR/Type.h"
70 #include "llvm/Support/Casting.h"
71 #include "llvm/Support/CodeGen.h"
73 #include "llvm/Support/Compiler.h"
75 #include "llvm/Support/KnownBits.h"
79 #include <cassert>
80 #include <cmath>
81 #include <cstdint>
82 #include <iterator>
83 #include <tuple>
84 #include <utility>
85 #include <vector>
86 
87 using namespace llvm;
88 
89 #define DEBUG_TYPE "si-lower"
90 
91 STATISTIC(NumTailCalls, "Number of tail calls");
92 
94  "amdgpu-vgpr-index-mode",
95  cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
96  cl::init(false));
97 
99  "amdgpu-disable-loop-alignment",
100  cl::desc("Do not align and prefetch loops"),
101  cl::init(false));
102 
103 static unsigned findFirstFreeSGPR(CCState &CCInfo) {
104  unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
105  for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
106  if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
107  return AMDGPU::SGPR0 + Reg;
108  }
109  }
110  llvm_unreachable("Cannot allocate sgpr");
111 }
112 
114  const GCNSubtarget &STI)
115  : AMDGPUTargetLowering(TM, STI),
116  Subtarget(&STI) {
117  addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
118  addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
119 
120  addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass);
121  addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass);
122 
123  addRegisterClass(MVT::f64, &AMDGPU::VReg_64RegClass);
124  addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
125  addRegisterClass(MVT::v2f32, &AMDGPU::VReg_64RegClass);
126 
127  addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass);
128  addRegisterClass(MVT::v3f32, &AMDGPU::VReg_96RegClass);
129 
130  addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass);
131  addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass);
132 
133  addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass);
134  addRegisterClass(MVT::v4f32, &AMDGPU::VReg_128RegClass);
135 
136  addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass);
137  addRegisterClass(MVT::v5f32, &AMDGPU::VReg_160RegClass);
138 
139  addRegisterClass(MVT::v8i32, &AMDGPU::SReg_256RegClass);
140  addRegisterClass(MVT::v8f32, &AMDGPU::VReg_256RegClass);
141 
142  addRegisterClass(MVT::v16i32, &AMDGPU::SReg_512RegClass);
143  addRegisterClass(MVT::v16f32, &AMDGPU::VReg_512RegClass);
144 
145  if (Subtarget->has16BitInsts()) {
146  addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass);
147  addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass);
148 
149  // Unless there are also VOP3P operations, not operations are really legal.
150  addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass);
151  addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass);
152  addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass);
153  addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass);
154  }
155 
156  if (Subtarget->hasMAIInsts()) {
157  addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
158  addRegisterClass(MVT::v32f32, &AMDGPU::VReg_1024RegClass);
159  }
160 
162 
163  // We need to custom lower vector stores from local memory
172 
181 
193 
196 
201 
207 
212 
215 
224 
231 
234 
237 
241 
242 #if 0
245 #endif
246 
247  // We only support LOAD/STORE and vector manipulation ops for vectors
248  // with > 4 elements.
252  for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
253  switch (Op) {
254  case ISD::LOAD:
255  case ISD::STORE:
256  case ISD::BUILD_VECTOR:
257  case ISD::BITCAST:
263  break;
264  case ISD::CONCAT_VECTORS:
266  break;
267  default:
269  break;
270  }
271  }
272  }
273 
275 
276  // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
277  // is expanded to avoid having two separate loops in case the index is a VGPR.
278 
279  // Most operations are naturally 32-bit vector operations. We only support
280  // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
281  for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) {
284 
287 
290 
293  }
294 
299 
302 
303  // Avoid stack access for these.
304  // TODO: Generalize to more vector types.
309 
315 
319 
324 
325  // Deal with vec3 vector operations when widened to vec4.
330 
331  // Deal with vec5 vector operations when widened to vec8.
336 
337  // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
338  // and output demarshalling
341 
342  // We can't return success/failure, only the old value,
343  // let LLVM add the comparison
346 
347  if (Subtarget->hasFlatAddressSpace()) {
350  }
351 
354 
355  // On SI this is s_memtime and s_memrealtime on VI.
359 
360  if (Subtarget->has16BitInsts()) {
364  }
365 
366  // v_mad_f32 does not support denormals according to some sources.
367  if (!Subtarget->hasFP32Denormals())
369 
370  if (!Subtarget->hasBFI()) {
371  // fcopysign can be done in a single instruction with BFI.
374  }
375 
376  if (!Subtarget->hasBCNT(32))
378 
379  if (!Subtarget->hasBCNT(64))
381 
382  if (Subtarget->hasFFBH())
384 
385  if (Subtarget->hasFFBL())
387 
388  // We only really have 32-bit BFE instructions (and 16-bit on VI).
389  //
390  // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
391  // effort to match them now. We want this to be false for i64 cases when the
392  // extraction isn't restricted to the upper or lower half. Ideally we would
393  // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
394  // span the midpoint are probably relatively rare, so don't worry about them
395  // for now.
396  if (Subtarget->hasBFE())
397  setHasExtractBitsInsn(true);
398 
403 
404 
405  // These are really only legal for ieee_mode functions. We should be avoiding
406  // them for functions that don't have ieee_mode enabled, so just say they are
407  // legal.
412 
413 
414  if (Subtarget->haveRoundOpsF64()) {
418  } else {
423  }
424 
426 
431 
432  if (Subtarget->has16BitInsts()) {
434 
437 
440 
443 
446 
451 
454 
460 
462 
464 
466 
468 
473 
478 
479  // F16 - Constant Actions.
481 
482  // F16 - Load/Store Actions.
487 
488  // F16 - VOP1 Actions.
497 
498  // F16 - VOP2 Actions.
501 
503 
504  // F16 - VOP3 Actions.
506  if (!Subtarget->hasFP16Denormals() && STI.hasMadF16())
508 
509  for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) {
510  for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
511  switch (Op) {
512  case ISD::LOAD:
513  case ISD::STORE:
514  case ISD::BUILD_VECTOR:
515  case ISD::BITCAST:
521  break;
522  case ISD::CONCAT_VECTORS:
524  break;
525  default:
527  break;
528  }
529  }
530  }
531 
532  // XXX - Do these do anything? Vector constants turn into build_vector.
535 
538 
543 
548 
555 
560 
565 
570 
574 
575  if (!Subtarget->hasVOP3PInsts()) {
578  }
579 
581  // This isn't really legal, but this avoids the legalizer unrolling it (and
582  // allows matching fneg (fabs x) patterns)
584 
589 
592 
595  }
596 
597  if (Subtarget->hasVOP3PInsts()) {
608 
612 
615 
617 
620 
623 
630 
635 
639 
642 
646 
650  }
651 
654 
655  if (Subtarget->has16BitInsts()) {
660  } else {
661  // Legalization hack.
664 
667  }
668 
671  }
672 
680 
690 
699 
727 
728  // All memory operations. Some folding on the pointer operand is done to help
729  // matching the constant offsets in the addressing modes.
748 
750 }
751 
753  return Subtarget;
754 }
755 
756 //===----------------------------------------------------------------------===//
757 // TargetLowering queries
758 //===----------------------------------------------------------------------===//
759 
760 // v_mad_mix* support a conversion from f16 to f32.
761 //
762 // There is only one special case when denormals are enabled we don't currently,
763 // where this is OK to use.
765  EVT DestVT, EVT SrcVT) const {
766  return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
767  (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
768  DestVT.getScalarType() == MVT::f32 && !Subtarget->hasFP32Denormals() &&
769  SrcVT.getScalarType() == MVT::f16;
770 }
771 
773  // SI has some legal vector types, but no legal vector operations. Say no
774  // shuffles are legal in order to prefer scalarizing some vector operations.
775  return false;
776 }
777 
779  CallingConv::ID CC,
780  EVT VT) const {
781  if (CC == CallingConv::AMDGPU_KERNEL)
782  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
783 
784  if (VT.isVector()) {
785  EVT ScalarVT = VT.getScalarType();
786  unsigned Size = ScalarVT.getSizeInBits();
787  if (Size == 32)
788  return ScalarVT.getSimpleVT();
789 
790  if (Size > 32)
791  return MVT::i32;
792 
793  if (Size == 16 && Subtarget->has16BitInsts())
794  return VT.isInteger() ? MVT::v2i16 : MVT::v2f16;
795  } else if (VT.getSizeInBits() > 32)
796  return MVT::i32;
797 
798  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
799 }
800 
802  CallingConv::ID CC,
803  EVT VT) const {
804  if (CC == CallingConv::AMDGPU_KERNEL)
805  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
806 
807  if (VT.isVector()) {
808  unsigned NumElts = VT.getVectorNumElements();
809  EVT ScalarVT = VT.getScalarType();
810  unsigned Size = ScalarVT.getSizeInBits();
811 
812  if (Size == 32)
813  return NumElts;
814 
815  if (Size > 32)
816  return NumElts * ((Size + 31) / 32);
817 
818  if (Size == 16 && Subtarget->has16BitInsts())
819  return (NumElts + 1) / 2;
820  } else if (VT.getSizeInBits() > 32)
821  return (VT.getSizeInBits() + 31) / 32;
822 
823  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
824 }
825 
828  EVT VT, EVT &IntermediateVT,
829  unsigned &NumIntermediates, MVT &RegisterVT) const {
830  if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
831  unsigned NumElts = VT.getVectorNumElements();
832  EVT ScalarVT = VT.getScalarType();
833  unsigned Size = ScalarVT.getSizeInBits();
834  if (Size == 32) {
835  RegisterVT = ScalarVT.getSimpleVT();
836  IntermediateVT = RegisterVT;
837  NumIntermediates = NumElts;
838  return NumIntermediates;
839  }
840 
841  if (Size > 32) {
842  RegisterVT = MVT::i32;
843  IntermediateVT = RegisterVT;
844  NumIntermediates = NumElts * ((Size + 31) / 32);
845  return NumIntermediates;
846  }
847 
848  // FIXME: We should fix the ABI to be the same on targets without 16-bit
849  // support, but unless we can properly handle 3-vectors, it will be still be
850  // inconsistent.
851  if (Size == 16 && Subtarget->has16BitInsts()) {
852  RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16;
853  IntermediateVT = RegisterVT;
854  NumIntermediates = (NumElts + 1) / 2;
855  return NumIntermediates;
856  }
857  }
858 
860  Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
861 }
862 
864  // Only limited forms of aggregate type currently expected.
865  assert(Ty->isStructTy() && "Expected struct type");
866 
867 
868  Type *ElementType = nullptr;
869  unsigned NumElts;
870  if (Ty->getContainedType(0)->isVectorTy()) {
871  VectorType *VecComponent = cast<VectorType>(Ty->getContainedType(0));
872  ElementType = VecComponent->getElementType();
873  NumElts = VecComponent->getNumElements();
874  } else {
875  ElementType = Ty->getContainedType(0);
876  NumElts = 1;
877  }
878 
879  assert((Ty->getContainedType(1) && Ty->getContainedType(1)->isIntegerTy(32)) && "Expected int32 type");
880 
881  // Calculate the size of the memVT type from the aggregate
882  unsigned Pow2Elts = 0;
883  unsigned ElementSize;
884  switch (ElementType->getTypeID()) {
885  default:
886  llvm_unreachable("Unknown type!");
887  case Type::IntegerTyID:
888  ElementSize = cast<IntegerType>(ElementType)->getBitWidth();
889  break;
890  case Type::HalfTyID:
891  ElementSize = 16;
892  break;
893  case Type::FloatTyID:
894  ElementSize = 32;
895  break;
896  }
897  unsigned AdditionalElts = ElementSize == 16 ? 2 : 1;
898  Pow2Elts = 1 << Log2_32_Ceil(NumElts + AdditionalElts);
899 
900  return MVT::getVectorVT(MVT::getVT(ElementType, false),
901  Pow2Elts);
902 }
903 
905  const CallInst &CI,
906  MachineFunction &MF,
907  unsigned IntrID) const {
908  if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
909  AMDGPU::lookupRsrcIntrinsic(IntrID)) {
911  (Intrinsic::ID)IntrID);
912  if (Attr.hasFnAttribute(Attribute::ReadNone))
913  return false;
914 
916 
917  if (RsrcIntr->IsImage) {
918  Info.ptrVal = MFI->getImagePSV(
920  CI.getArgOperand(RsrcIntr->RsrcArg));
921  Info.align.reset();
922  } else {
923  Info.ptrVal = MFI->getBufferPSV(
925  CI.getArgOperand(RsrcIntr->RsrcArg));
926  }
927 
929  if (Attr.hasFnAttribute(Attribute::ReadOnly)) {
931  Info.memVT = MVT::getVT(CI.getType(), true);
932  if (Info.memVT == MVT::Other) {
933  // Some intrinsics return an aggregate type - special case to work out
934  // the correct memVT
935  Info.memVT = memVTFromAggregate(CI.getType());
936  }
938  } else if (Attr.hasFnAttribute(Attribute::WriteOnly)) {
939  Info.opc = ISD::INTRINSIC_VOID;
940  Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
942  } else {
943  // Atomic
945  Info.memVT = MVT::getVT(CI.getType());
949 
950  // XXX - Should this be volatile without known ordering?
952  }
953  return true;
954  }
955 
956  switch (IntrID) {
957  case Intrinsic::amdgcn_atomic_inc:
958  case Intrinsic::amdgcn_atomic_dec:
959  case Intrinsic::amdgcn_ds_ordered_add:
960  case Intrinsic::amdgcn_ds_ordered_swap:
961  case Intrinsic::amdgcn_ds_fadd:
962  case Intrinsic::amdgcn_ds_fmin:
963  case Intrinsic::amdgcn_ds_fmax: {
965  Info.memVT = MVT::getVT(CI.getType());
966  Info.ptrVal = CI.getOperand(0);
967  Info.align.reset();
969 
970  const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
971  if (!Vol->isZero())
973 
974  return true;
975  }
976  case Intrinsic::amdgcn_buffer_atomic_fadd: {
978 
979  Info.opc = ISD::INTRINSIC_VOID;
980  Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
981  Info.ptrVal = MFI->getBufferPSV(
983  CI.getArgOperand(1));
984  Info.align.reset();
986 
987  const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4));
988  if (!Vol || !Vol->isZero())
990 
991  return true;
992  }
993  case Intrinsic::amdgcn_global_atomic_fadd: {
994  Info.opc = ISD::INTRINSIC_VOID;
995  Info.memVT = MVT::getVT(CI.getOperand(0)->getType()
997  Info.ptrVal = CI.getOperand(0);
998  Info.align.reset();
1000 
1001  return true;
1002  }
1003  case Intrinsic::amdgcn_ds_append:
1004  case Intrinsic::amdgcn_ds_consume: {
1005  Info.opc = ISD::INTRINSIC_W_CHAIN;
1006  Info.memVT = MVT::getVT(CI.getType());
1007  Info.ptrVal = CI.getOperand(0);
1008  Info.align.reset();
1010 
1011  const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
1012  if (!Vol->isZero())
1014 
1015  return true;
1016  }
1017  case Intrinsic::amdgcn_ds_gws_init:
1018  case Intrinsic::amdgcn_ds_gws_barrier:
1019  case Intrinsic::amdgcn_ds_gws_sema_v:
1020  case Intrinsic::amdgcn_ds_gws_sema_br:
1021  case Intrinsic::amdgcn_ds_gws_sema_p:
1022  case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1023  Info.opc = ISD::INTRINSIC_VOID;
1024 
1026  Info.ptrVal =
1028 
1029  // This is an abstract access, but we need to specify a type and size.
1030  Info.memVT = MVT::i32;
1031  Info.size = 4;
1032  Info.align = Align(4);
1033 
1035  if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1037  return true;
1038  }
1039  default:
1040  return false;
1041  }
1042 }
1043 
1046  Type *&AccessTy) const {
1047  switch (II->getIntrinsicID()) {
1048  case Intrinsic::amdgcn_atomic_inc:
1049  case Intrinsic::amdgcn_atomic_dec:
1050  case Intrinsic::amdgcn_ds_ordered_add:
1051  case Intrinsic::amdgcn_ds_ordered_swap:
1052  case Intrinsic::amdgcn_ds_fadd:
1053  case Intrinsic::amdgcn_ds_fmin:
1054  case Intrinsic::amdgcn_ds_fmax: {
1055  Value *Ptr = II->getArgOperand(0);
1056  AccessTy = II->getType();
1057  Ops.push_back(Ptr);
1058  return true;
1059  }
1060  default:
1061  return false;
1062  }
1063 }
1064 
1065 bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const {
1066  if (!Subtarget->hasFlatInstOffsets()) {
1067  // Flat instructions do not have offsets, and only have the register
1068  // address.
1069  return AM.BaseOffs == 0 && AM.Scale == 0;
1070  }
1071 
1072  // GFX9 added a 13-bit signed offset. When using regular flat instructions,
1073  // the sign bit is ignored and is treated as a 12-bit unsigned offset.
1074 
1075  // GFX10 shrinked signed offset to 12 bits. When using regular flat
1076  // instructions, the sign bit is also ignored and is treated as 11-bit
1077  // unsigned offset.
1078 
1079  if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
1080  return isUInt<11>(AM.BaseOffs) && AM.Scale == 0;
1081 
1082  // Just r + i
1083  return isUInt<12>(AM.BaseOffs) && AM.Scale == 0;
1084 }
1085 
1087  if (Subtarget->hasFlatGlobalInsts())
1088  return isInt<13>(AM.BaseOffs) && AM.Scale == 0;
1089 
1090  if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
1091  // Assume the we will use FLAT for all global memory accesses
1092  // on VI.
1093  // FIXME: This assumption is currently wrong. On VI we still use
1094  // MUBUF instructions for the r + i addressing mode. As currently
1095  // implemented, the MUBUF instructions only work on buffer < 4GB.
1096  // It may be possible to support > 4GB buffers with MUBUF instructions,
1097  // by setting the stride value in the resource descriptor which would
1098  // increase the size limit to (stride * 4GB). However, this is risky,
1099  // because it has never been validated.
1100  return isLegalFlatAddressingMode(AM);
1101  }
1102 
1103  return isLegalMUBUFAddressingMode(AM);
1104 }
1105 
1106 bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
1107  // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
1108  // additionally can do r + r + i with addr64. 32-bit has more addressing
1109  // mode options. Depending on the resource constant, it can also do
1110  // (i64 r0) + (i32 r1) * (i14 i).
1111  //
1112  // Private arrays end up using a scratch buffer most of the time, so also
1113  // assume those use MUBUF instructions. Scratch loads / stores are currently
1114  // implemented as mubuf instructions with offen bit set, so slightly
1115  // different than the normal addr64.
1116  if (!isUInt<12>(AM.BaseOffs))
1117  return false;
1118 
1119  // FIXME: Since we can split immediate into soffset and immediate offset,
1120  // would it make sense to allow any immediate?
1121 
1122  switch (AM.Scale) {
1123  case 0: // r + i or just i, depending on HasBaseReg.
1124  return true;
1125  case 1:
1126  return true; // We have r + r or r + i.
1127  case 2:
1128  if (AM.HasBaseReg) {
1129  // Reject 2 * r + r.
1130  return false;
1131  }
1132 
1133  // Allow 2 * r as r + r
1134  // Or 2 * r + i is allowed as r + r + i.
1135  return true;
1136  default: // Don't allow n * r
1137  return false;
1138  }
1139 }
1140 
1142  const AddrMode &AM, Type *Ty,
1143  unsigned AS, Instruction *I) const {
1144  // No global is ever allowed as a base.
1145  if (AM.BaseGV)
1146  return false;
1147 
1148  if (AS == AMDGPUAS::GLOBAL_ADDRESS)
1149  return isLegalGlobalAddressingMode(AM);
1150 
1151  if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
1154  // If the offset isn't a multiple of 4, it probably isn't going to be
1155  // correctly aligned.
1156  // FIXME: Can we get the real alignment here?
1157  if (AM.BaseOffs % 4 != 0)
1158  return isLegalMUBUFAddressingMode(AM);
1159 
1160  // There are no SMRD extloads, so if we have to do a small type access we
1161  // will use a MUBUF load.
1162  // FIXME?: We also need to do this if unaligned, but we don't know the
1163  // alignment here.
1164  if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
1165  return isLegalGlobalAddressingMode(AM);
1166 
1167  if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
1168  // SMRD instructions have an 8-bit, dword offset on SI.
1169  if (!isUInt<8>(AM.BaseOffs / 4))
1170  return false;
1171  } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
1172  // On CI+, this can also be a 32-bit literal constant offset. If it fits
1173  // in 8-bits, it can use a smaller encoding.
1174  if (!isUInt<32>(AM.BaseOffs / 4))
1175  return false;
1176  } else if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
1177  // On VI, these use the SMEM format and the offset is 20-bit in bytes.
1178  if (!isUInt<20>(AM.BaseOffs))
1179  return false;
1180  } else
1181  llvm_unreachable("unhandled generation");
1182 
1183  if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
1184  return true;
1185 
1186  if (AM.Scale == 1 && AM.HasBaseReg)
1187  return true;
1188 
1189  return false;
1190 
1191  } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
1192  return isLegalMUBUFAddressingMode(AM);
1193  } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||
1194  AS == AMDGPUAS::REGION_ADDRESS) {
1195  // Basic, single offset DS instructions allow a 16-bit unsigned immediate
1196  // field.
1197  // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
1198  // an 8-bit dword offset but we don't know the alignment here.
1199  if (!isUInt<16>(AM.BaseOffs))
1200  return false;
1201 
1202  if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
1203  return true;
1204 
1205  if (AM.Scale == 1 && AM.HasBaseReg)
1206  return true;
1207 
1208  return false;
1209  } else if (AS == AMDGPUAS::FLAT_ADDRESS ||
1211  // For an unknown address space, this usually means that this is for some
1212  // reason being used for pure arithmetic, and not based on some addressing
1213  // computation. We don't have instructions that compute pointers with any
1214  // addressing modes, so treat them as having no offset like flat
1215  // instructions.
1216  return isLegalFlatAddressingMode(AM);
1217  } else {
1218  llvm_unreachable("unhandled address space");
1219  }
1220 }
1221 
1222 bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
1223  const SelectionDAG &DAG) const {
1224  if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
1225  return (MemVT.getSizeInBits() <= 4 * 32);
1226  } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
1227  unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
1228  return (MemVT.getSizeInBits() <= MaxPrivateBits);
1229  } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
1230  return (MemVT.getSizeInBits() <= 2 * 32);
1231  }
1232  return true;
1233 }
1234 
1236  unsigned Size, unsigned AddrSpace, unsigned Align,
1237  MachineMemOperand::Flags Flags, bool *IsFast) const {
1238  if (IsFast)
1239  *IsFast = false;
1240 
1241  if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
1242  AddrSpace == AMDGPUAS::REGION_ADDRESS) {
1243  // ds_read/write_b64 require 8-byte alignment, but we can do a 4 byte
1244  // aligned, 8 byte access in a single operation using ds_read2/write2_b32
1245  // with adjacent offsets.
1246  bool AlignedBy4 = (Align % 4 == 0);
1247  if (IsFast)
1248  *IsFast = AlignedBy4;
1249 
1250  return AlignedBy4;
1251  }
1252 
1253  // FIXME: We have to be conservative here and assume that flat operations
1254  // will access scratch. If we had access to the IR function, then we
1255  // could determine if any private memory was used in the function.
1256  if (!Subtarget->hasUnalignedScratchAccess() &&
1257  (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
1258  AddrSpace == AMDGPUAS::FLAT_ADDRESS)) {
1259  bool AlignedBy4 = Align >= 4;
1260  if (IsFast)
1261  *IsFast = AlignedBy4;
1262 
1263  return AlignedBy4;
1264  }
1265 
1266  if (Subtarget->hasUnalignedBufferAccess()) {
1267  // If we have an uniform constant load, it still requires using a slow
1268  // buffer instruction if unaligned.
1269  if (IsFast) {
1270  *IsFast = (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
1271  AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ?
1272  (Align % 4 == 0) : true;
1273  }
1274 
1275  return true;
1276  }
1277 
1278  // Smaller than dword value must be aligned.
1279  if (Size < 32)
1280  return false;
1281 
1282  // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
1283  // byte-address are ignored, thus forcing Dword alignment.
1284  // This applies to private, global, and constant memory.
1285  if (IsFast)
1286  *IsFast = true;
1287 
1288  return Size >= 32 && Align >= 4;
1289 }
1290 
1292  EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
1293  bool *IsFast) const {
1294  if (IsFast)
1295  *IsFast = false;
1296 
1297  // TODO: I think v3i32 should allow unaligned accesses on CI with DS_READ_B96,
1298  // which isn't a simple VT.
1299  // Until MVT is extended to handle this, simply check for the size and
1300  // rely on the condition below: allow accesses if the size is a multiple of 4.
1301  if (VT == MVT::Other || (VT != MVT::Other && VT.getSizeInBits() > 1024 &&
1302  VT.getStoreSize() > 16)) {
1303  return false;
1304  }
1305 
1306  return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace,
1307  Align, Flags, IsFast);
1308 }
1309 
1311  uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset,
1312  bool ZeroMemset, bool MemcpyStrSrc,
1313  const AttributeList &FuncAttributes) const {
1314  // FIXME: Should account for address space here.
1315 
1316  // The default fallback uses the private pointer size as a guess for a type to
1317  // use. Make sure we switch these to 64-bit accesses.
1318 
1319  if (Size >= 16 && DstAlign >= 4) // XXX: Should only do for global
1320  return MVT::v4i32;
1321 
1322  if (Size >= 8 && DstAlign >= 4)
1323  return MVT::v2i32;
1324 
1325  // Use the default.
1326  return MVT::Other;
1327 }
1328 
1329 static bool isFlatGlobalAddrSpace(unsigned AS) {
1330  return AS == AMDGPUAS::GLOBAL_ADDRESS ||
1331  AS == AMDGPUAS::FLAT_ADDRESS ||
1334 }
1335 
1337  unsigned DestAS) const {
1338  return isFlatGlobalAddrSpace(SrcAS) && isFlatGlobalAddrSpace(DestAS);
1339 }
1340 
1342  const MemSDNode *MemNode = cast<MemSDNode>(N);
1343  const Value *Ptr = MemNode->getMemOperand()->getValue();
1344  const Instruction *I = dyn_cast_or_null<Instruction>(Ptr);
1345  return I && I->getMetadata("amdgpu.noclobber");
1346 }
1347 
1349  unsigned DestAS) const {
1350  // Flat -> private/local is a simple truncate.
1351  // Flat -> global is no-op
1352  if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
1353  return true;
1354 
1355  return isNoopAddrSpaceCast(SrcAS, DestAS);
1356 }
1357 
1359  const MemSDNode *MemNode = cast<MemSDNode>(N);
1360 
1361  return AMDGPUInstrInfo::isUniformMMO(MemNode->getMemOperand());
1362 }
1363 
1366  int NumElts = VT.getVectorNumElements();
1367  if (NumElts != 1 && VT.getScalarType().bitsLE(MVT::i16))
1370 }
1371 
1373  Type *Ty) const {
1374  // FIXME: Could be smarter if called for vector constants.
1375  return true;
1376 }
1377 
1379  if (Subtarget->has16BitInsts() && VT == MVT::i16) {
1380  switch (Op) {
1381  case ISD::LOAD:
1382  case ISD::STORE:
1383 
1384  // These operations are done with 32-bit instructions anyway.
1385  case ISD::AND:
1386  case ISD::OR:
1387  case ISD::XOR:
1388  case ISD::SELECT:
1389  // TODO: Extensions?
1390  return true;
1391  default:
1392  return false;
1393  }
1394  }
1395 
1396  // SimplifySetCC uses this function to determine whether or not it should
1397  // create setcc with i1 operands. We don't have instructions for i1 setcc.
1398  if (VT == MVT::i1 && Op == ISD::SETCC)
1399  return false;
1400 
1401  return TargetLowering::isTypeDesirableForOp(Op, VT);
1402 }
1403 
1404 SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
1405  const SDLoc &SL,
1406  SDValue Chain,
1407  uint64_t Offset) const {
1408  const DataLayout &DL = DAG.getDataLayout();
1409  MachineFunction &MF = DAG.getMachineFunction();
1411 
1412  const ArgDescriptor *InputPtrReg;
1413  const TargetRegisterClass *RC;
1414 
1415  std::tie(InputPtrReg, RC)
1417 
1420  SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
1421  MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
1422 
1423  return DAG.getObjectPtrOffset(SL, BasePtr, Offset);
1424 }
1425 
1426 SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
1427  const SDLoc &SL) const {
1428  uint64_t Offset = getImplicitParameterOffset(DAG.getMachineFunction(),
1429  FIRST_IMPLICIT);
1430  return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
1431 }
1432 
1433 SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
1434  const SDLoc &SL, SDValue Val,
1435  bool Signed,
1436  const ISD::InputArg *Arg) const {
1437  // First, if it is a widened vector, narrow it.
1438  if (VT.isVector() &&
1439  VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
1440  EVT NarrowedVT =
1442  VT.getVectorNumElements());
1443  Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
1444  DAG.getConstant(0, SL, MVT::i32));
1445  }
1446 
1447  // Then convert the vector elements or scalar value.
1448  if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) &&
1449  VT.bitsLT(MemVT)) {
1450  unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
1451  Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
1452  }
1453 
1454  if (MemVT.isFloatingPoint())
1455  Val = getFPExtOrFPTrunc(DAG, Val, SL, VT);
1456  else if (Signed)
1457  Val = DAG.getSExtOrTrunc(Val, SL, VT);
1458  else
1459  Val = DAG.getZExtOrTrunc(Val, SL, VT);
1460 
1461  return Val;
1462 }
1463 
1464 SDValue SITargetLowering::lowerKernargMemParameter(
1465  SelectionDAG &DAG, EVT VT, EVT MemVT,
1466  const SDLoc &SL, SDValue Chain,
1467  uint64_t Offset, unsigned Align, bool Signed,
1468  const ISD::InputArg *Arg) const {
1469  Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
1471  MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
1472 
1473  // Try to avoid using an extload by loading earlier than the argument address,
1474  // and extracting the relevant bits. The load should hopefully be merged with
1475  // the previous argument.
1476  if (MemVT.getStoreSize() < 4 && Align < 4) {
1477  // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
1478  int64_t AlignDownOffset = alignDown(Offset, 4);
1479  int64_t OffsetDiff = Offset - AlignDownOffset;
1480 
1481  EVT IntVT = MemVT.changeTypeToInteger();
1482 
1483  // TODO: If we passed in the base kernel offset we could have a better
1484  // alignment than 4, but we don't really need it.
1485  SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
1486  SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, 4,
1489 
1490  SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
1491  SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
1492 
1493  SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
1494  ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
1495  ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
1496 
1497 
1498  return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL);
1499  }
1500 
1501  SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
1502  SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Align,
1505 
1506  SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
1507  return DAG.getMergeValues({ Val, Load.getValue(1) }, SL);
1508 }
1509 
1510 SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
1511  const SDLoc &SL, SDValue Chain,
1512  const ISD::InputArg &Arg) const {
1513  MachineFunction &MF = DAG.getMachineFunction();
1514  MachineFrameInfo &MFI = MF.getFrameInfo();
1515 
1516  if (Arg.Flags.isByVal()) {
1517  unsigned Size = Arg.Flags.getByValSize();
1518  int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false);
1519  return DAG.getFrameIndex(FrameIdx, MVT::i32);
1520  }
1521 
1522  unsigned ArgOffset = VA.getLocMemOffset();
1523  unsigned ArgSize = VA.getValVT().getStoreSize();
1524 
1525  int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true);
1526 
1527  // Create load nodes to retrieve arguments from the stack.
1528  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1529  SDValue ArgValue;
1530 
1531  // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
1533  MVT MemVT = VA.getValVT();
1534 
1535  switch (VA.getLocInfo()) {
1536  default:
1537  break;
1538  case CCValAssign::BCvt:
1539  MemVT = VA.getLocVT();
1540  break;
1541  case CCValAssign::SExt:
1542  ExtType = ISD::SEXTLOAD;
1543  break;
1544  case CCValAssign::ZExt:
1545  ExtType = ISD::ZEXTLOAD;
1546  break;
1547  case CCValAssign::AExt:
1548  ExtType = ISD::EXTLOAD;
1549  break;
1550  }
1551 
1552  ArgValue = DAG.getExtLoad(
1553  ExtType, SL, VA.getLocVT(), Chain, FIN,
1555  MemVT);
1556  return ArgValue;
1557 }
1558 
1559 SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
1560  const SIMachineFunctionInfo &MFI,
1561  EVT VT,
1563  const ArgDescriptor *Reg;
1564  const TargetRegisterClass *RC;
1565 
1566  std::tie(Reg, RC) = MFI.getPreloadedValue(PVID);
1567  return CreateLiveInRegister(DAG, RC, Reg->getRegister(), VT);
1568 }
1569 
1571  CallingConv::ID CallConv,
1573  BitVector &Skipped,
1574  FunctionType *FType,
1576  for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
1577  const ISD::InputArg *Arg = &Ins[I];
1578 
1579  assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
1580  "vector type argument should have been split");
1581 
1582  // First check if it's a PS input addr.
1583  if (CallConv == CallingConv::AMDGPU_PS &&
1584  !Arg->Flags.isInReg() && PSInputNum <= 15) {
1585  bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum);
1586 
1587  // Inconveniently only the first part of the split is marked as isSplit,
1588  // so skip to the end. We only want to increment PSInputNum once for the
1589  // entire split argument.
1590  if (Arg->Flags.isSplit()) {
1591  while (!Arg->Flags.isSplitEnd()) {
1592  assert((!Arg->VT.isVector() ||
1593  Arg->VT.getScalarSizeInBits() == 16) &&
1594  "unexpected vector split in ps argument type");
1595  if (!SkipArg)
1596  Splits.push_back(*Arg);
1597  Arg = &Ins[++I];
1598  }
1599  }
1600 
1601  if (SkipArg) {
1602  // We can safely skip PS inputs.
1603  Skipped.set(Arg->getOrigArgIndex());
1604  ++PSInputNum;
1605  continue;
1606  }
1607 
1608  Info->markPSInputAllocated(PSInputNum);
1609  if (Arg->Used)
1610  Info->markPSInputEnabled(PSInputNum);
1611 
1612  ++PSInputNum;
1613  }
1614 
1615  Splits.push_back(*Arg);
1616  }
1617 }
1618 
1619 // Allocate special inputs passed in VGPRs.
1621  MachineFunction &MF,
1622  const SIRegisterInfo &TRI,
1623  SIMachineFunctionInfo &Info) const {
1624  const LLT S32 = LLT::scalar(32);
1626 
1627  if (Info.hasWorkItemIDX()) {
1628  Register Reg = AMDGPU::VGPR0;
1629  MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1630 
1631  CCInfo.AllocateReg(Reg);
1633  }
1634 
1635  if (Info.hasWorkItemIDY()) {
1636  Register Reg = AMDGPU::VGPR1;
1637  MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1638 
1639  CCInfo.AllocateReg(Reg);
1641  }
1642 
1643  if (Info.hasWorkItemIDZ()) {
1644  Register Reg = AMDGPU::VGPR2;
1645  MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1646 
1647  CCInfo.AllocateReg(Reg);
1649  }
1650 }
1651 
1652 // Try to allocate a VGPR at the end of the argument list, or if no argument
1653 // VGPRs are left allocating a stack slot.
1654 // If \p Mask is is given it indicates bitfield position in the register.
1655 // If \p Arg is given use it with new ]p Mask instead of allocating new.
1656 static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
1657  ArgDescriptor Arg = ArgDescriptor()) {
1658  if (Arg.isSet())
1659  return ArgDescriptor::createArg(Arg, Mask);
1660 
1661  ArrayRef<MCPhysReg> ArgVGPRs
1662  = makeArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
1663  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs);
1664  if (RegIdx == ArgVGPRs.size()) {
1665  // Spill to stack required.
1666  int64_t Offset = CCInfo.AllocateStack(4, 4);
1667 
1668  return ArgDescriptor::createStack(Offset, Mask);
1669  }
1670 
1671  unsigned Reg = ArgVGPRs[RegIdx];
1672  Reg = CCInfo.AllocateReg(Reg);
1673  assert(Reg != AMDGPU::NoRegister);
1674 
1675  MachineFunction &MF = CCInfo.getMachineFunction();
1676  Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
1677  MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32));
1678  return ArgDescriptor::createRegister(Reg, Mask);
1679 }
1680 
1682  const TargetRegisterClass *RC,
1683  unsigned NumArgRegs) {
1684  ArrayRef<MCPhysReg> ArgSGPRs = makeArrayRef(RC->begin(), 32);
1685  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
1686  if (RegIdx == ArgSGPRs.size())
1687  report_fatal_error("ran out of SGPRs for arguments");
1688 
1689  unsigned Reg = ArgSGPRs[RegIdx];
1690  Reg = CCInfo.AllocateReg(Reg);
1691  assert(Reg != AMDGPU::NoRegister);
1692 
1693  MachineFunction &MF = CCInfo.getMachineFunction();
1694  MF.addLiveIn(Reg, RC);
1695  return ArgDescriptor::createRegister(Reg);
1696 }
1697 
1699  return allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32);
1700 }
1701 
1703  return allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
1704 }
1705 
1707  MachineFunction &MF,
1708  const SIRegisterInfo &TRI,
1709  SIMachineFunctionInfo &Info) const {
1710  const unsigned Mask = 0x3ff;
1712 
1713  if (Info.hasWorkItemIDX()) {
1714  Arg = allocateVGPR32Input(CCInfo, Mask);
1715  Info.setWorkItemIDX(Arg);
1716  }
1717 
1718  if (Info.hasWorkItemIDY()) {
1719  Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg);
1720  Info.setWorkItemIDY(Arg);
1721  }
1722 
1723  if (Info.hasWorkItemIDZ())
1724  Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg));
1725 }
1726 
1728  CCState &CCInfo,
1729  MachineFunction &MF,
1730  const SIRegisterInfo &TRI,
1731  SIMachineFunctionInfo &Info) const {
1732  auto &ArgInfo = Info.getArgInfo();
1733 
1734  // TODO: Unify handling with private memory pointers.
1735 
1736  if (Info.hasDispatchPtr())
1737  ArgInfo.DispatchPtr = allocateSGPR64Input(CCInfo);
1738 
1739  if (Info.hasQueuePtr())
1740  ArgInfo.QueuePtr = allocateSGPR64Input(CCInfo);
1741 
1742  if (Info.hasKernargSegmentPtr())
1743  ArgInfo.KernargSegmentPtr = allocateSGPR64Input(CCInfo);
1744 
1745  if (Info.hasDispatchID())
1746  ArgInfo.DispatchID = allocateSGPR64Input(CCInfo);
1747 
1748  // flat_scratch_init is not applicable for non-kernel functions.
1749 
1750  if (Info.hasWorkGroupIDX())
1751  ArgInfo.WorkGroupIDX = allocateSGPR32Input(CCInfo);
1752 
1753  if (Info.hasWorkGroupIDY())
1754  ArgInfo.WorkGroupIDY = allocateSGPR32Input(CCInfo);
1755 
1756  if (Info.hasWorkGroupIDZ())
1757  ArgInfo.WorkGroupIDZ = allocateSGPR32Input(CCInfo);
1758 
1759  if (Info.hasImplicitArgPtr())
1760  ArgInfo.ImplicitArgPtr = allocateSGPR64Input(CCInfo);
1761 }
1762 
1763 // Allocate special inputs passed in user SGPRs.
1765  MachineFunction &MF,
1766  const SIRegisterInfo &TRI,
1767  SIMachineFunctionInfo &Info) const {
1768  if (Info.hasImplicitBufferPtr()) {
1769  unsigned ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
1770  MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
1771  CCInfo.AllocateReg(ImplicitBufferPtrReg);
1772  }
1773 
1774  // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
1775  if (Info.hasPrivateSegmentBuffer()) {
1776  unsigned PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
1777  MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
1778  CCInfo.AllocateReg(PrivateSegmentBufferReg);
1779  }
1780 
1781  if (Info.hasDispatchPtr()) {
1782  unsigned DispatchPtrReg = Info.addDispatchPtr(TRI);
1783  MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
1784  CCInfo.AllocateReg(DispatchPtrReg);
1785  }
1786 
1787  if (Info.hasQueuePtr()) {
1788  unsigned QueuePtrReg = Info.addQueuePtr(TRI);
1789  MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
1790  CCInfo.AllocateReg(QueuePtrReg);
1791  }
1792 
1793  if (Info.hasKernargSegmentPtr()) {
1795  Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
1796  CCInfo.AllocateReg(InputPtrReg);
1797 
1798  Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass);
1800  }
1801 
1802  if (Info.hasDispatchID()) {
1803  unsigned DispatchIDReg = Info.addDispatchID(TRI);
1804  MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
1805  CCInfo.AllocateReg(DispatchIDReg);
1806  }
1807 
1808  if (Info.hasFlatScratchInit()) {
1809  unsigned FlatScratchInitReg = Info.addFlatScratchInit(TRI);
1810  MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
1811  CCInfo.AllocateReg(FlatScratchInitReg);
1812  }
1813 
1814  // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
1815  // these from the dispatch pointer.
1816 }
1817 
1818 // Allocate special input registers that are initialized per-wave.
1820  MachineFunction &MF,
1822  CallingConv::ID CallConv,
1823  bool IsShader) const {
1824  if (Info.hasWorkGroupIDX()) {
1825  unsigned Reg = Info.addWorkGroupIDX();
1826  MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
1827  CCInfo.AllocateReg(Reg);
1828  }
1829 
1830  if (Info.hasWorkGroupIDY()) {
1831  unsigned Reg = Info.addWorkGroupIDY();
1832  MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
1833  CCInfo.AllocateReg(Reg);
1834  }
1835 
1836  if (Info.hasWorkGroupIDZ()) {
1837  unsigned Reg = Info.addWorkGroupIDZ();
1838  MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
1839  CCInfo.AllocateReg(Reg);
1840  }
1841 
1842  if (Info.hasWorkGroupInfo()) {
1843  unsigned Reg = Info.addWorkGroupInfo();
1844  MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
1845  CCInfo.AllocateReg(Reg);
1846  }
1847 
1848  if (Info.hasPrivateSegmentWaveByteOffset()) {
1849  // Scratch wave offset passed in system SGPR.
1850  unsigned PrivateSegmentWaveByteOffsetReg;
1851 
1852  if (IsShader) {
1853  PrivateSegmentWaveByteOffsetReg =
1855 
1856  // This is true if the scratch wave byte offset doesn't have a fixed
1857  // location.
1858  if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
1859  PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
1860  Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
1861  }
1862  } else
1863  PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
1864 
1865  MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
1866  CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
1867  }
1868 }
1869 
1871  MachineFunction &MF,
1872  const SIRegisterInfo &TRI,
1874  // Now that we've figured out where the scratch register inputs are, see if
1875  // should reserve the arguments and use them directly.
1876  MachineFrameInfo &MFI = MF.getFrameInfo();
1877  bool HasStackObjects = MFI.hasStackObjects();
1878  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1879 
1880  // Record that we know we have non-spill stack objects so we don't need to
1881  // check all stack objects later.
1882  if (HasStackObjects)
1883  Info.setHasNonSpillStackObjects(true);
1884 
1885  // Everything live out of a block is spilled with fast regalloc, so it's
1886  // almost certain that spilling will be required.
1887  if (TM.getOptLevel() == CodeGenOpt::None)
1888  HasStackObjects = true;
1889 
1890  // For now assume stack access is needed in any callee functions, so we need
1891  // the scratch registers to pass in.
1892  bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
1893 
1894  if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) {
1895  // If we have stack objects, we unquestionably need the private buffer
1896  // resource. For the Code Object V2 ABI, this will be the first 4 user
1897  // SGPR inputs. We can reserve those and use them directly.
1898 
1899  Register PrivateSegmentBufferReg =
1901  Info.setScratchRSrcReg(PrivateSegmentBufferReg);
1902  } else {
1903  unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
1904  // We tentatively reserve the last registers (skipping the last registers
1905  // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
1906  // we'll replace these with the ones immediately after those which were
1907  // really allocated. In the prologue copies will be inserted from the
1908  // argument to these reserved registers.
1909 
1910  // Without HSA, relocations are used for the scratch pointer and the
1911  // buffer resource setup is always inserted in the prologue. Scratch wave
1912  // offset is still in an input SGPR.
1913  Info.setScratchRSrcReg(ReservedBufferReg);
1914  }
1915 
1916  // hasFP should be accurate for kernels even before the frame is finalized.
1917  if (ST.getFrameLowering()->hasFP(MF)) {
1919 
1920  // Try to use s32 as the SP, but move it if it would interfere with input
1921  // arguments. This won't work with calls though.
1922  //
1923  // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
1924  // registers.
1925  if (!MRI.isLiveIn(AMDGPU::SGPR32)) {
1926  Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
1927  } else {
1929 
1930  if (MFI.hasCalls())
1931  report_fatal_error("call in graphics shader with too many input SGPRs");
1932 
1933  for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
1934  if (!MRI.isLiveIn(Reg)) {
1935  Info.setStackPtrOffsetReg(Reg);
1936  break;
1937  }
1938  }
1939 
1940  if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
1941  report_fatal_error("failed to find register for SP");
1942  }
1943 
1944  if (MFI.hasCalls()) {
1945  Info.setScratchWaveOffsetReg(AMDGPU::SGPR33);
1946  Info.setFrameOffsetReg(AMDGPU::SGPR33);
1947  } else {
1948  unsigned ReservedOffsetReg =
1950  Info.setScratchWaveOffsetReg(ReservedOffsetReg);
1951  Info.setFrameOffsetReg(ReservedOffsetReg);
1952  }
1953  } else if (RequiresStackAccess) {
1954  assert(!MFI.hasCalls());
1955  // We know there are accesses and they will be done relative to SP, so just
1956  // pin it to the input.
1957  //
1958  // FIXME: Should not do this if inline asm is reading/writing these
1959  // registers.
1960  Register PreloadedSP = Info.getPreloadedReg(
1962 
1963  Info.setStackPtrOffsetReg(PreloadedSP);
1964  Info.setScratchWaveOffsetReg(PreloadedSP);
1965  Info.setFrameOffsetReg(PreloadedSP);
1966  } else {
1967  assert(!MFI.hasCalls());
1968 
1969  // There may not be stack access at all. There may still be spills, or
1970  // access of a constant pointer (in which cases an extra copy will be
1971  // emitted in the prolog).
1972  unsigned ReservedOffsetReg
1974  Info.setStackPtrOffsetReg(ReservedOffsetReg);
1975  Info.setScratchWaveOffsetReg(ReservedOffsetReg);
1976  Info.setFrameOffsetReg(ReservedOffsetReg);
1977  }
1978 }
1979 
1982  return !Info->isEntryFunction();
1983 }
1984 
1986 
1987 }
1988 
1991  const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
1993 
1994  const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
1995  if (!IStart)
1996  return;
1997 
1998  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1999  MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
2000  MachineBasicBlock::iterator MBBI = Entry->begin();
2001  for (const MCPhysReg *I = IStart; *I; ++I) {
2002  const TargetRegisterClass *RC = nullptr;
2003  if (AMDGPU::SReg_64RegClass.contains(*I))
2004  RC = &AMDGPU::SGPR_64RegClass;
2005  else if (AMDGPU::SReg_32RegClass.contains(*I))
2006  RC = &AMDGPU::SGPR_32RegClass;
2007  else
2008  llvm_unreachable("Unexpected register class in CSRsViaCopy!");
2009 
2010  Register NewVR = MRI->createVirtualRegister(RC);
2011  // Create copy from CSR to a virtual register.
2012  Entry->addLiveIn(*I);
2013  BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
2014  .addReg(*I);
2015 
2016  // Insert the copy-back instructions right before the terminator.
2017  for (auto *Exit : Exits)
2018  BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
2019  TII->get(TargetOpcode::COPY), *I)
2020  .addReg(NewVR);
2021  }
2022 }
2023 
2025  SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
2026  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2027  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2029 
2030  MachineFunction &MF = DAG.getMachineFunction();
2031  const Function &Fn = MF.getFunction();
2032  FunctionType *FType = MF.getFunction().getFunctionType();
2034 
2035  if (Subtarget->isAmdHsaOS() && AMDGPU::isShader(CallConv)) {
2036  DiagnosticInfoUnsupported NoGraphicsHSA(
2037  Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc());
2038  DAG.getContext()->diagnose(NoGraphicsHSA);
2039  return DAG.getEntryNode();
2040  }
2041 
2044  BitVector Skipped(Ins.size());
2045  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2046  *DAG.getContext());
2047 
2048  bool IsShader = AMDGPU::isShader(CallConv);
2049  bool IsKernel = AMDGPU::isKernel(CallConv);
2050  bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv);
2051 
2052  if (IsShader) {
2053  processShaderInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
2054 
2055  // At least one interpolation mode must be enabled or else the GPU will
2056  // hang.
2057  //
2058  // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
2059  // set PSInputAddr, the user wants to enable some bits after the compilation
2060  // based on run-time states. Since we can't know what the final PSInputEna
2061  // will look like, so we shouldn't do anything here and the user should take
2062  // responsibility for the correct programming.
2063  //
2064  // Otherwise, the following restrictions apply:
2065  // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
2066  // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
2067  // enabled too.
2068  if (CallConv == CallingConv::AMDGPU_PS) {
2069  if ((Info->getPSInputAddr() & 0x7F) == 0 ||
2070  ((Info->getPSInputAddr() & 0xF) == 0 &&
2071  Info->isPSInputAllocated(11))) {
2072  CCInfo.AllocateReg(AMDGPU::VGPR0);
2073  CCInfo.AllocateReg(AMDGPU::VGPR1);
2074  Info->markPSInputAllocated(0);
2075  Info->markPSInputEnabled(0);
2076  }
2077  if (Subtarget->isAmdPalOS()) {
2078  // For isAmdPalOS, the user does not enable some bits after compilation
2079  // based on run-time states; the register values being generated here are
2080  // the final ones set in hardware. Therefore we need to apply the
2081  // workaround to PSInputAddr and PSInputEnable together. (The case where
2082  // a bit is set in PSInputAddr but not PSInputEnable is where the
2083  // frontend set up an input arg for a particular interpolation mode, but
2084  // nothing uses that input arg. Really we should have an earlier pass
2085  // that removes such an arg.)
2086  unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
2087  if ((PsInputBits & 0x7F) == 0 ||
2088  ((PsInputBits & 0xF) == 0 &&
2089  (PsInputBits >> 11 & 1)))
2090  Info->markPSInputEnabled(
2092  }
2093  }
2094 
2095  assert(!Info->hasDispatchPtr() &&
2096  !Info->hasKernargSegmentPtr() && !Info->hasFlatScratchInit() &&
2097  !Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
2098  !Info->hasWorkGroupIDZ() && !Info->hasWorkGroupInfo() &&
2099  !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() &&
2100  !Info->hasWorkItemIDZ());
2101  } else if (IsKernel) {
2102  assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
2103  } else {
2104  Splits.append(Ins.begin(), Ins.end());
2105  }
2106 
2107  if (IsEntryFunc) {
2108  allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
2109  allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
2110  }
2111 
2112  if (IsKernel) {
2113  analyzeFormalArgumentsCompute(CCInfo, Ins);
2114  } else {
2115  CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
2116  CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
2117  }
2118 
2119  SmallVector<SDValue, 16> Chains;
2120 
2121  // FIXME: This is the minimum kernel argument alignment. We should improve
2122  // this to the maximum alignment of the arguments.
2123  //
2124  // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
2125  // kern arg offset.
2126  const unsigned KernelArgBaseAlign = 16;
2127 
2128  for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
2129  const ISD::InputArg &Arg = Ins[i];
2130  if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) {
2131  InVals.push_back(DAG.getUNDEF(Arg.VT));
2132  continue;
2133  }
2134 
2135  CCValAssign &VA = ArgLocs[ArgIdx++];
2136  MVT VT = VA.getLocVT();
2137 
2138  if (IsEntryFunc && VA.isMemLoc()) {
2139  VT = Ins[i].VT;
2140  EVT MemVT = VA.getLocVT();
2141 
2142  const uint64_t Offset = VA.getLocMemOffset();
2143  unsigned Align = MinAlign(KernelArgBaseAlign, Offset);
2144 
2145  SDValue Arg = lowerKernargMemParameter(
2146  DAG, VT, MemVT, DL, Chain, Offset, Align, Ins[i].Flags.isSExt(), &Ins[i]);
2147  Chains.push_back(Arg.getValue(1));
2148 
2149  auto *ParamTy =
2150  dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
2151  if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
2152  ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
2153  ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
2154  // On SI local pointers are just offsets into LDS, so they are always
2155  // less than 16-bits. On CI and newer they could potentially be
2156  // real pointers, so we can't guarantee their size.
2157  Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg,
2158  DAG.getValueType(MVT::i16));
2159  }
2160 
2161  InVals.push_back(Arg);
2162  continue;
2163  } else if (!IsEntryFunc && VA.isMemLoc()) {
2164  SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
2165  InVals.push_back(Val);
2166  if (!Arg.Flags.isByVal())
2167  Chains.push_back(Val.getValue(1));
2168  continue;
2169  }
2170 
2171  assert(VA.isRegLoc() && "Parameter must be in a register!");
2172 
2173  Register Reg = VA.getLocReg();
2174  const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2175  EVT ValVT = VA.getValVT();
2176 
2177  Reg = MF.addLiveIn(Reg, RC);
2178  SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
2179 
2180  if (Arg.Flags.isSRet()) {
2181  // The return object should be reasonably addressable.
2182 
2183  // FIXME: This helps when the return is a real sret. If it is a
2184  // automatically inserted sret (i.e. CanLowerReturn returns false), an
2185  // extra copy is inserted in SelectionDAGBuilder which obscures this.
2186  unsigned NumBits
2188  Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
2189  DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits)));
2190  }
2191 
2192  // If this is an 8 or 16-bit value, it is really passed promoted
2193  // to 32 bits. Insert an assert[sz]ext to capture this, then
2194  // truncate to the right size.
2195  switch (VA.getLocInfo()) {
2196  case CCValAssign::Full:
2197  break;
2198  case CCValAssign::BCvt:
2199  Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val);
2200  break;
2201  case CCValAssign::SExt:
2202  Val = DAG.getNode(ISD::AssertSext, DL, VT, Val,
2203  DAG.getValueType(ValVT));
2204  Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2205  break;
2206  case CCValAssign::ZExt:
2207  Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
2208  DAG.getValueType(ValVT));
2209  Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2210  break;
2211  case CCValAssign::AExt:
2212  Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2213  break;
2214  default:
2215  llvm_unreachable("Unknown loc info!");
2216  }
2217 
2218  InVals.push_back(Val);
2219  }
2220 
2221  if (!IsEntryFunc) {
2222  // Special inputs come after user arguments.
2223  allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
2224  }
2225 
2226  // Start adding system SGPRs.
2227  if (IsEntryFunc) {
2228  allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsShader);
2229  } else {
2230  CCInfo.AllocateReg(Info->getScratchRSrcReg());
2231  CCInfo.AllocateReg(Info->getScratchWaveOffsetReg());
2232  CCInfo.AllocateReg(Info->getFrameOffsetReg());
2233  allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
2234  }
2235 
2236  auto &ArgUsageInfo =
2238  ArgUsageInfo.setFuncArgInfo(Fn, Info->getArgInfo());
2239 
2240  unsigned StackArgSize = CCInfo.getNextStackOffset();
2241  Info->setBytesInStackArgArea(StackArgSize);
2242 
2243  return Chains.empty() ? Chain :
2244  DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
2245 }
2246 
2247 // TODO: If return values can't fit in registers, we should return as many as
2248 // possible in registers before passing on stack.
2250  CallingConv::ID CallConv,
2251  MachineFunction &MF, bool IsVarArg,
2252  const SmallVectorImpl<ISD::OutputArg> &Outs,
2253  LLVMContext &Context) const {
2254  // Replacing returns with sret/stack usage doesn't make sense for shaders.
2255  // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
2256  // for shaders. Vector types should be explicitly handled by CC.
2257  if (AMDGPU::isEntryFunctionCC(CallConv))
2258  return true;
2259 
2261  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2262  return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg));
2263 }
2264 
2265 SDValue
2267  bool isVarArg,
2268  const SmallVectorImpl<ISD::OutputArg> &Outs,
2269  const SmallVectorImpl<SDValue> &OutVals,
2270  const SDLoc &DL, SelectionDAG &DAG) const {
2271  MachineFunction &MF = DAG.getMachineFunction();
2273 
2274  if (AMDGPU::isKernel(CallConv)) {
2275  return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
2276  OutVals, DL, DAG);
2277  }
2278 
2279  bool IsShader = AMDGPU::isShader(CallConv);
2280 
2281  Info->setIfReturnsVoid(Outs.empty());
2282  bool IsWaveEnd = Info->returnsVoid() && IsShader;
2283 
2284  // CCValAssign - represent the assignment of the return value to a location.
2287 
2288  // CCState - Info about the registers and stack slots.
2289  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2290  *DAG.getContext());
2291 
2292  // Analyze outgoing return values.
2293  CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
2294 
2295  SDValue Flag;
2296  SmallVector<SDValue, 48> RetOps;
2297  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2298 
2299  // Add return address for callable functions.
2300  if (!Info->isEntryFunction()) {
2302  SDValue ReturnAddrReg = CreateLiveInRegister(
2303  DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64);
2304 
2305  SDValue ReturnAddrVirtualReg = DAG.getRegister(
2306  MF.getRegInfo().createVirtualRegister(&AMDGPU::CCR_SGPR_64RegClass),
2307  MVT::i64);
2308  Chain =
2309  DAG.getCopyToReg(Chain, DL, ReturnAddrVirtualReg, ReturnAddrReg, Flag);
2310  Flag = Chain.getValue(1);
2311  RetOps.push_back(ReturnAddrVirtualReg);
2312  }
2313 
2314  // Copy the result values into the output registers.
2315  for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
2316  ++I, ++RealRVLocIdx) {
2317  CCValAssign &VA = RVLocs[I];
2318  assert(VA.isRegLoc() && "Can only return in registers!");
2319  // TODO: Partially return in registers if return values don't fit.
2320  SDValue Arg = OutVals[RealRVLocIdx];
2321 
2322  // Copied from other backends.
2323  switch (VA.getLocInfo()) {
2324  case CCValAssign::Full:
2325  break;
2326  case CCValAssign::BCvt:
2327  Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
2328  break;
2329  case CCValAssign::SExt:
2330  Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
2331  break;
2332  case CCValAssign::ZExt:
2333  Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
2334  break;
2335  case CCValAssign::AExt:
2336  Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
2337  break;
2338  default:
2339  llvm_unreachable("Unknown loc info!");
2340  }
2341 
2342  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag);
2343  Flag = Chain.getValue(1);
2344  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2345  }
2346 
2347  // FIXME: Does sret work properly?
2348  if (!Info->isEntryFunction()) {
2349  const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2350  const MCPhysReg *I =
2352  if (I) {
2353  for (; *I; ++I) {
2354  if (AMDGPU::SReg_64RegClass.contains(*I))
2355  RetOps.push_back(DAG.getRegister(*I, MVT::i64));
2356  else if (AMDGPU::SReg_32RegClass.contains(*I))
2357  RetOps.push_back(DAG.getRegister(*I, MVT::i32));
2358  else
2359  llvm_unreachable("Unexpected register class in CSRsViaCopy!");
2360  }
2361  }
2362  }
2363 
2364  // Update chain and glue.
2365  RetOps[0] = Chain;
2366  if (Flag.getNode())
2367  RetOps.push_back(Flag);
2368 
2369  unsigned Opc = AMDGPUISD::ENDPGM;
2370  if (!IsWaveEnd)
2372  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
2373 }
2374 
2376  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
2377  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2378  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
2379  SDValue ThisVal) const {
2380  CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg);
2381 
2382  // Assign locations to each value returned by this call.
2384  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2385  *DAG.getContext());
2386  CCInfo.AnalyzeCallResult(Ins, RetCC);
2387 
2388  // Copy all of the result registers out of their specified physreg.
2389  for (unsigned i = 0; i != RVLocs.size(); ++i) {
2390  CCValAssign VA = RVLocs[i];
2391  SDValue Val;
2392 
2393  if (VA.isRegLoc()) {
2394  Val = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
2395  Chain = Val.getValue(1);
2396  InFlag = Val.getValue(2);
2397  } else if (VA.isMemLoc()) {
2398  report_fatal_error("TODO: return values in memory");
2399  } else
2400  llvm_unreachable("unknown argument location type");
2401 
2402  switch (VA.getLocInfo()) {
2403  case CCValAssign::Full:
2404  break;
2405  case CCValAssign::BCvt:
2406  Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
2407  break;
2408  case CCValAssign::ZExt:
2409  Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
2410  DAG.getValueType(VA.getValVT()));
2411  Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2412  break;
2413  case CCValAssign::SExt:
2414  Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
2415  DAG.getValueType(VA.getValVT()));
2416  Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2417  break;
2418  case CCValAssign::AExt:
2419  Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2420  break;
2421  default:
2422  llvm_unreachable("Unknown loc info!");
2423  }
2424 
2425  InVals.push_back(Val);
2426  }
2427 
2428  return Chain;
2429 }
2430 
2431 // Add code to pass special inputs required depending on used features separate
2432 // from the explicit user arguments present in the IR.
2434  CallLoweringInfo &CLI,
2435  CCState &CCInfo,
2436  const SIMachineFunctionInfo &Info,
2437  SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
2438  SmallVectorImpl<SDValue> &MemOpChains,
2439  SDValue Chain) const {
2440  // If we don't have a call site, this was a call inserted by
2441  // legalization. These can never use special inputs.
2442  if (!CLI.CS)
2443  return;
2444 
2445  const Function *CalleeFunc = CLI.CS.getCalledFunction();
2446  assert(CalleeFunc);
2447 
2448  SelectionDAG &DAG = CLI.DAG;
2449  const SDLoc &DL = CLI.DL;
2450 
2451  const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2452 
2453  auto &ArgUsageInfo =
2455  const AMDGPUFunctionArgInfo &CalleeArgInfo
2456  = ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc);
2457 
2458  const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
2459 
2460  // TODO: Unify with private memory register handling. This is complicated by
2461  // the fact that at least in kernels, the input argument is not necessarily
2462  // in the same location as the input.
2472  };
2473 
2474  for (auto InputID : InputRegs) {
2475  const ArgDescriptor *OutgoingArg;
2476  const TargetRegisterClass *ArgRC;
2477 
2478  std::tie(OutgoingArg, ArgRC) = CalleeArgInfo.getPreloadedValue(InputID);
2479  if (!OutgoingArg)
2480  continue;
2481 
2482  const ArgDescriptor *IncomingArg;
2483  const TargetRegisterClass *IncomingArgRC;
2484  std::tie(IncomingArg, IncomingArgRC)
2485  = CallerArgInfo.getPreloadedValue(InputID);
2486  assert(IncomingArgRC == ArgRC);
2487 
2488  // All special arguments are ints for now.
2489  EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
2490  SDValue InputReg;
2491 
2492  if (IncomingArg) {
2493  InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg);
2494  } else {
2495  // The implicit arg ptr is special because it doesn't have a corresponding
2496  // input for kernels, and is computed from the kernarg segment pointer.
2497  assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
2498  InputReg = getImplicitArgPtr(DAG, DL);
2499  }
2500 
2501  if (OutgoingArg->isRegister()) {
2502  RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
2503  } else {
2504  unsigned SpecialArgOffset = CCInfo.AllocateStack(ArgVT.getStoreSize(), 4);
2505  SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2506  SpecialArgOffset);
2507  MemOpChains.push_back(ArgStore);
2508  }
2509  }
2510 
2511  // Pack workitem IDs into a single register or pass it as is if already
2512  // packed.
2513  const ArgDescriptor *OutgoingArg;
2514  const TargetRegisterClass *ArgRC;
2515 
2516  std::tie(OutgoingArg, ArgRC) =
2517  CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
2518  if (!OutgoingArg)
2519  std::tie(OutgoingArg, ArgRC) =
2520  CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
2521  if (!OutgoingArg)
2522  std::tie(OutgoingArg, ArgRC) =
2523  CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
2524  if (!OutgoingArg)
2525  return;
2526 
2527  const ArgDescriptor *IncomingArgX
2529  const ArgDescriptor *IncomingArgY
2531  const ArgDescriptor *IncomingArgZ
2533 
2534  SDValue InputReg;
2535  SDLoc SL;
2536 
2537  // If incoming ids are not packed we need to pack them.
2538  if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX)
2539  InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
2540 
2541  if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY) {
2542  SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
2543  Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
2544  DAG.getShiftAmountConstant(10, MVT::i32, SL));
2545  InputReg = InputReg.getNode() ?
2546  DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y;
2547  }
2548 
2549  if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ) {
2550  SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
2551  Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
2552  DAG.getShiftAmountConstant(20, MVT::i32, SL));
2553  InputReg = InputReg.getNode() ?
2554  DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z;
2555  }
2556 
2557  if (!InputReg.getNode()) {
2558  // Workitem ids are already packed, any of present incoming arguments
2559  // will carry all required fields.
2561  IncomingArgX ? *IncomingArgX :
2562  IncomingArgY ? *IncomingArgY :
2563  *IncomingArgZ, ~0u);
2564  InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
2565  }
2566 
2567  if (OutgoingArg->isRegister()) {
2568  RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
2569  } else {
2570  unsigned SpecialArgOffset = CCInfo.AllocateStack(4, 4);
2571  SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2572  SpecialArgOffset);
2573  MemOpChains.push_back(ArgStore);
2574  }
2575 }
2576 
2578  return CC == CallingConv::Fast;
2579 }
2580 
2581 /// Return true if we might ever do TCO for calls with this calling convention.
2583  switch (CC) {
2584  case CallingConv::C:
2585  return true;
2586  default:
2587  return canGuaranteeTCO(CC);
2588  }
2589 }
2590 
2592  SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
2593  const SmallVectorImpl<ISD::OutputArg> &Outs,
2594  const SmallVectorImpl<SDValue> &OutVals,
2595  const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
2596  if (!mayTailCallThisCC(CalleeCC))
2597  return false;
2598 
2599  MachineFunction &MF = DAG.getMachineFunction();
2600  const Function &CallerF = MF.getFunction();
2601  CallingConv::ID CallerCC = CallerF.getCallingConv();
2603  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2604 
2605  // Kernels aren't callable, and don't have a live in return address so it
2606  // doesn't make sense to do a tail call with entry functions.
2607  if (!CallerPreserved)
2608  return false;
2609 
2610  bool CCMatch = CallerCC == CalleeCC;
2611 
2613  if (canGuaranteeTCO(CalleeCC) && CCMatch)
2614  return true;
2615  return false;
2616  }
2617 
2618  // TODO: Can we handle var args?
2619  if (IsVarArg)
2620  return false;
2621 
2622  for (const Argument &Arg : CallerF.args()) {
2623  if (Arg.hasByValAttr())
2624  return false;
2625  }
2626 
2627  LLVMContext &Ctx = *DAG.getContext();
2628 
2629  // Check that the call results are passed in the same way.
2630  if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins,
2631  CCAssignFnForCall(CalleeCC, IsVarArg),
2632  CCAssignFnForCall(CallerCC, IsVarArg)))
2633  return false;
2634 
2635  // The callee has to preserve all registers the caller needs to preserve.
2636  if (!CCMatch) {
2637  const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2638  if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
2639  return false;
2640  }
2641 
2642  // Nothing more to check if the callee is taking no arguments.
2643  if (Outs.empty())
2644  return true;
2645 
2647  CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
2648 
2649  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
2650 
2651  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
2652  // If the stack arguments for this call do not fit into our own save area then
2653  // the call cannot be made tail.
2654  // TODO: Is this really necessary?
2655  if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
2656  return false;
2657 
2658  const MachineRegisterInfo &MRI = MF.getRegInfo();
2659  return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
2660 }
2661 
2663  if (!CI->isTailCall())
2664  return false;
2665 
2666  const Function *ParentFn = CI->getParent()->getParent();
2667  if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv()))
2668  return false;
2669 
2670  auto Attr = ParentFn->getFnAttribute("disable-tail-calls");
2671  return (Attr.getValueAsString() != "true");
2672 }
2673 
2674 // The wave scratch offset register is used as the global base pointer.
2676  SmallVectorImpl<SDValue> &InVals) const {
2677  SelectionDAG &DAG = CLI.DAG;
2678  const SDLoc &DL = CLI.DL;
2680  SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
2682  SDValue Chain = CLI.Chain;
2683  SDValue Callee = CLI.Callee;
2684  bool &IsTailCall = CLI.IsTailCall;
2685  CallingConv::ID CallConv = CLI.CallConv;
2686  bool IsVarArg = CLI.IsVarArg;
2687  bool IsSibCall = false;
2688  bool IsThisReturn = false;
2689  MachineFunction &MF = DAG.getMachineFunction();
2690 
2691  if (Callee.isUndef() || isNullConstant(Callee)) {
2692  if (!CLI.IsTailCall) {
2693  for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I)
2694  InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT));
2695  }
2696 
2697  return Chain;
2698  }
2699 
2700  if (IsVarArg) {
2701  return lowerUnhandledCall(CLI, InVals,
2702  "unsupported call to variadic function ");
2703  }
2704 
2705  if (!CLI.CS.getInstruction())
2706  report_fatal_error("unsupported libcall legalization");
2707 
2708  if (!CLI.CS.getCalledFunction()) {
2709  return lowerUnhandledCall(CLI, InVals,
2710  "unsupported indirect call to function ");
2711  }
2712 
2713  if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
2714  return lowerUnhandledCall(CLI, InVals,
2715  "unsupported required tail call to function ");
2716  }
2717 
2719  // Note the issue is with the CC of the calling function, not of the call
2720  // itself.
2721  return lowerUnhandledCall(CLI, InVals,
2722  "unsupported call from graphics shader of function ");
2723  }
2724 
2725  if (IsTailCall) {
2726  IsTailCall = isEligibleForTailCallOptimization(
2727  Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
2728  if (!IsTailCall && CLI.CS && CLI.CS.isMustTailCall()) {
2729  report_fatal_error("failed to perform tail call elimination on a call "
2730  "site marked musttail");
2731  }
2732 
2733  bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
2734 
2735  // A sibling call is one where we're under the usual C ABI and not planning
2736  // to change that but can still do a tail call:
2737  if (!TailCallOpt && IsTailCall)
2738  IsSibCall = true;
2739 
2740  if (IsTailCall)
2741  ++NumTailCalls;
2742  }
2743 
2745 
2746  // Analyze operands of the call, assigning locations to each operand.
2748  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2749  CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
2750 
2751  CCInfo.AnalyzeCallOperands(Outs, AssignFn);
2752 
2753  // Get a count of how many bytes are to be pushed on the stack.
2754  unsigned NumBytes = CCInfo.getNextStackOffset();
2755 
2756  if (IsSibCall) {
2757  // Since we're not changing the ABI to make this a tail call, the memory
2758  // operands are already available in the caller's incoming argument space.
2759  NumBytes = 0;
2760  }
2761 
2762  // FPDiff is the byte offset of the call's argument area from the callee's.
2763  // Stores to callee stack arguments will be placed in FixedStackSlots offset
2764  // by this amount for a tail call. In a sibling call it must be 0 because the
2765  // caller will deallocate the entire stack and the callee still expects its
2766  // arguments to begin at SP+0. Completely unused for non-tail calls.
2767  int32_t FPDiff = 0;
2768  MachineFrameInfo &MFI = MF.getFrameInfo();
2770 
2771  // Adjust the stack pointer for the new arguments...
2772  // These operations are automatically eliminated by the prolog/epilog pass
2773  if (!IsSibCall) {
2774  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
2775 
2776  SmallVector<SDValue, 4> CopyFromChains;
2777 
2778  // In the HSA case, this should be an identity copy.
2779  SDValue ScratchRSrcReg
2780  = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
2781  RegsToPass.emplace_back(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
2782  CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
2783  Chain = DAG.getTokenFactor(DL, CopyFromChains);
2784  }
2785 
2786  SmallVector<SDValue, 8> MemOpChains;
2787  MVT PtrVT = MVT::i32;
2788 
2789  // Walk the register/memloc assignments, inserting copies/loads.
2790  for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e;
2791  ++i, ++realArgIdx) {
2792  CCValAssign &VA = ArgLocs[i];
2793  SDValue Arg = OutVals[realArgIdx];
2794 
2795  // Promote the value if needed.
2796  switch (VA.getLocInfo()) {
2797  case CCValAssign::Full:
2798  break;
2799  case CCValAssign::BCvt:
2800  Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
2801  break;
2802  case CCValAssign::ZExt:
2803  Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
2804  break;
2805  case CCValAssign::SExt:
2806  Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
2807  break;
2808  case CCValAssign::AExt:
2809  Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
2810  break;
2811  case CCValAssign::FPExt:
2812  Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
2813  break;
2814  default:
2815  llvm_unreachable("Unknown loc info!");
2816  }
2817 
2818  if (VA.isRegLoc()) {
2819  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2820  } else {
2821  assert(VA.isMemLoc());
2822 
2823  SDValue DstAddr;
2824  MachinePointerInfo DstInfo;
2825 
2826  unsigned LocMemOffset = VA.getLocMemOffset();
2827  int32_t Offset = LocMemOffset;
2828 
2829  SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
2830  MaybeAlign Alignment;
2831 
2832  if (IsTailCall) {
2833  ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2834  unsigned OpSize = Flags.isByVal() ?
2835  Flags.getByValSize() : VA.getValVT().getStoreSize();
2836 
2837  // FIXME: We can have better than the minimum byval required alignment.
2838  Alignment =
2839  Flags.isByVal()
2840  ? MaybeAlign(Flags.getByValAlign())
2841  : commonAlignment(Subtarget->getStackAlignment(), Offset);
2842 
2843  Offset = Offset + FPDiff;
2844  int FI = MFI.CreateFixedObject(OpSize, Offset, true);
2845 
2846  DstAddr = DAG.getFrameIndex(FI, PtrVT);
2847  DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
2848 
2849  // Make sure any stack arguments overlapping with where we're storing
2850  // are loaded before this eventual operation. Otherwise they'll be
2851  // clobbered.
2852 
2853  // FIXME: Why is this really necessary? This seems to just result in a
2854  // lot of code to copy the stack and write them back to the same
2855  // locations, which are supposed to be immutable?
2856  Chain = addTokenForArgument(Chain, DAG, MFI, FI);
2857  } else {
2858  DstAddr = PtrOff;
2859  DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
2860  Alignment =
2861  commonAlignment(Subtarget->getStackAlignment(), LocMemOffset);
2862  }
2863 
2864  if (Outs[i].Flags.isByVal()) {
2865  SDValue SizeNode =
2866  DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);
2867  SDValue Cpy = DAG.getMemcpy(
2868  Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(),
2869  /*isVol = */ false, /*AlwaysInline = */ true,
2870  /*isTailCall = */ false, DstInfo,
2873 
2874  MemOpChains.push_back(Cpy);
2875  } else {
2876  SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo,
2877  Alignment ? Alignment->value() : 0);
2878  MemOpChains.push_back(Store);
2879  }
2880  }
2881  }
2882 
2883  // Copy special input registers after user input arguments.
2884  passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
2885 
2886  if (!MemOpChains.empty())
2887  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2888 
2889  // Build a sequence of copy-to-reg nodes chained together with token chain
2890  // and flag operands which copy the outgoing args into the appropriate regs.
2891  SDValue InFlag;
2892  for (auto &RegToPass : RegsToPass) {
2893  Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
2894  RegToPass.second, InFlag);
2895  InFlag = Chain.getValue(1);
2896  }
2897 
2898 
2899  SDValue PhysReturnAddrReg;
2900  if (IsTailCall) {
2901  // Since the return is being combined with the call, we need to pass on the
2902  // return address.
2903 
2905  SDValue ReturnAddrReg = CreateLiveInRegister(
2906  DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64);
2907 
2908  PhysReturnAddrReg = DAG.getRegister(TRI->getReturnAddressReg(MF),
2909  MVT::i64);
2910  Chain = DAG.getCopyToReg(Chain, DL, PhysReturnAddrReg, ReturnAddrReg, InFlag);
2911  InFlag = Chain.getValue(1);
2912  }
2913 
2914  // We don't usually want to end the call-sequence here because we would tidy
2915  // the frame up *after* the call, however in the ABI-changing tail-call case
2916  // we've carefully laid out the parameters so that when sp is reset they'll be
2917  // in the correct location.
2918  if (IsTailCall && !IsSibCall) {
2919  Chain = DAG.getCALLSEQ_END(Chain,
2920  DAG.getTargetConstant(NumBytes, DL, MVT::i32),
2921  DAG.getTargetConstant(0, DL, MVT::i32),
2922  InFlag, DL);
2923  InFlag = Chain.getValue(1);
2924  }
2925 
2926  std::vector<SDValue> Ops;
2927  Ops.push_back(Chain);
2928  Ops.push_back(Callee);
2929  // Add a redundant copy of the callee global which will not be legalized, as
2930  // we need direct access to the callee later.
2931  GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Callee);
2932  const GlobalValue *GV = GSD->getGlobal();
2933  Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
2934 
2935  if (IsTailCall) {
2936  // Each tail call may have to adjust the stack by a different amount, so
2937  // this information must travel along with the operation for eventual
2938  // consumption by emitEpilogue.
2939  Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
2940 
2941  Ops.push_back(PhysReturnAddrReg);
2942  }
2943 
2944  // Add argument registers to the end of the list so that they are known live
2945  // into the call.
2946  for (auto &RegToPass : RegsToPass) {
2947  Ops.push_back(DAG.getRegister(RegToPass.first,
2948  RegToPass.second.getValueType()));
2949  }
2950 
2951  // Add a register mask operand representing the call-preserved registers.
2952 
2953  auto *TRI = static_cast<const SIRegisterInfo*>(Subtarget->getRegisterInfo());
2954  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2955  assert(Mask && "Missing call preserved mask for calling convention");
2956  Ops.push_back(DAG.getRegisterMask(Mask));
2957 
2958  if (InFlag.getNode())
2959  Ops.push_back(InFlag);
2960 
2961  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2962 
2963  // If we're doing a tall call, use a TC_RETURN here rather than an
2964  // actual call instruction.
2965  if (IsTailCall) {
2966  MFI.setHasTailCall();
2967  return DAG.getNode(AMDGPUISD::TC_RETURN, DL, NodeTys, Ops);
2968  }
2969 
2970  // Returns a chain and a flag for retval copy to use.
2971  SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops);
2972  Chain = Call.getValue(0);
2973  InFlag = Call.getValue(1);
2974 
2975  uint64_t CalleePopBytes = NumBytes;
2976  Chain = DAG.getCALLSEQ_END(Chain, DAG.getTargetConstant(0, DL, MVT::i32),
2977  DAG.getTargetConstant(CalleePopBytes, DL, MVT::i32),
2978  InFlag, DL);
2979  if (!Ins.empty())
2980  InFlag = Chain.getValue(1);
2981 
2982  // Handle result values, copying them out of physregs into vregs that we
2983  // return.
2984  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
2985  InVals, IsThisReturn,
2986  IsThisReturn ? OutVals[0] : SDValue());
2987 }
2988 
2990  const MachineFunction &MF) const {
2992  .Case("m0", AMDGPU::M0)
2993  .Case("exec", AMDGPU::EXEC)
2994  .Case("exec_lo", AMDGPU::EXEC_LO)
2995  .Case("exec_hi", AMDGPU::EXEC_HI)
2996  .Case("flat_scratch", AMDGPU::FLAT_SCR)
2997  .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2998  .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2999  .Default(Register());
3000 
3001  if (Reg == AMDGPU::NoRegister) {
3002  report_fatal_error(Twine("invalid register name \""
3003  + StringRef(RegName) + "\"."));
3004 
3005  }
3006 
3007  if (!Subtarget->hasFlatScrRegister() &&
3008  Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) {
3009  report_fatal_error(Twine("invalid register \""
3010  + StringRef(RegName) + "\" for subtarget."));
3011  }
3012 
3013  switch (Reg) {
3014  case AMDGPU::M0:
3015  case AMDGPU::EXEC_LO:
3016  case AMDGPU::EXEC_HI:
3017  case AMDGPU::FLAT_SCR_LO:
3018  case AMDGPU::FLAT_SCR_HI:
3019  if (VT.getSizeInBits() == 32)
3020  return Reg;
3021  break;
3022  case AMDGPU::EXEC:
3023  case AMDGPU::FLAT_SCR:
3024  if (VT.getSizeInBits() == 64)
3025  return Reg;
3026  break;
3027  default:
3028  llvm_unreachable("missing register type checking");
3029  }
3030 
3031  report_fatal_error(Twine("invalid type for register \""
3032  + StringRef(RegName) + "\"."));
3033 }
3034 
3035 // If kill is not the last instruction, split the block so kill is always a
3036 // proper terminator.
3038  MachineBasicBlock *BB) const {
3039  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3040 
3041  MachineBasicBlock::iterator SplitPoint(&MI);
3042  ++SplitPoint;
3043 
3044  if (SplitPoint == BB->end()) {
3045  // Don't bother with a new block.
3047  return BB;
3048  }
3049 
3050  MachineFunction *MF = BB->getParent();
3051  MachineBasicBlock *SplitBB
3053 
3054  MF->insert(++MachineFunction::iterator(BB), SplitBB);
3055  SplitBB->splice(SplitBB->begin(), BB, SplitPoint, BB->end());
3056 
3057  SplitBB->transferSuccessorsAndUpdatePHIs(BB);
3058  BB->addSuccessor(SplitBB);
3059 
3061  return SplitBB;
3062 }
3063 
3064 // Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
3065 // \p MI will be the only instruction in the loop body block. Otherwise, it will
3066 // be the first instruction in the remainder block.
3067 //
3068 /// \returns { LoopBody, Remainder }
3069 static std::pair<MachineBasicBlock *, MachineBasicBlock *>
3071  MachineFunction *MF = MBB.getParent();
3073 
3074  // To insert the loop we need to split the block. Move everything after this
3075  // point to a new block, and insert a new empty block between the two.
3077  MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
3078  MachineFunction::iterator MBBI(MBB);
3079  ++MBBI;
3080 
3081  MF->insert(MBBI, LoopBB);
3082  MF->insert(MBBI, RemainderBB);
3083 
3084  LoopBB->addSuccessor(LoopBB);
3085  LoopBB->addSuccessor(RemainderBB);
3086 
3087  // Move the rest of the block into a new block.
3088  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
3089 
3090  if (InstInLoop) {
3091  auto Next = std::next(I);
3092 
3093  // Move instruction to loop body.
3094  LoopBB->splice(LoopBB->begin(), &MBB, I, Next);
3095 
3096  // Move the rest of the block.
3097  RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end());
3098  } else {
3099  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
3100  }
3101 
3102  MBB.addSuccessor(LoopBB);
3103 
3104  return std::make_pair(LoopBB, RemainderBB);
3105 }
3106 
3107 /// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
3109  MachineBasicBlock *MBB = MI.getParent();
3110  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3111  auto I = MI.getIterator();
3112  auto E = std::next(I);
3113 
3114  BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
3115  .addImm(0);
3116 
3117  MIBundleBuilder Bundler(*MBB, I, E);
3118  finalizeBundle(*MBB, Bundler.begin());
3119 }
3120 
3123  MachineBasicBlock *BB) const {
3124  const DebugLoc &DL = MI.getDebugLoc();
3125 
3127 
3128  MachineBasicBlock *LoopBB;
3129  MachineBasicBlock *RemainderBB;
3130  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3131 
3132  // Apparently kill flags are only valid if the def is in the same block?
3133  if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0))
3134  Src->setIsKill(false);
3135 
3136  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true);
3137 
3138  MachineBasicBlock::iterator I = LoopBB->end();
3139 
3140  const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg(
3142 
3143  // Clear TRAP_STS.MEM_VIOL
3144  BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
3145  .addImm(0)
3146  .addImm(EncodedReg);
3147 
3149 
3150  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3151 
3152  // Load and check TRAP_STS.MEM_VIOL
3153  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
3154  .addImm(EncodedReg);
3155 
3156  // FIXME: Do we need to use an isel pseudo that may clobber scc?
3157  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32))
3158  .addReg(Reg, RegState::Kill)
3159  .addImm(0);
3160  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
3161  .addMBB(LoopBB);
3162 
3163  return RemainderBB;
3164 }
3165 
3166 // Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
3167 // wavefront. If the value is uniform and just happens to be in a VGPR, this
3168 // will only do one iteration. In the worst case, this will loop 64 times.
3169 //
3170 // TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
3172  const SIInstrInfo *TII,
3174  MachineBasicBlock &OrigBB,
3175  MachineBasicBlock &LoopBB,
3176  const DebugLoc &DL,
3177  const MachineOperand &IdxReg,
3178  unsigned InitReg,
3179  unsigned ResultReg,
3180  unsigned PhiReg,
3181  unsigned InitSaveExecReg,
3182  int Offset,
3183  bool UseGPRIdxMode,
3184  bool IsIndirectSrc) {
3185  MachineFunction *MF = OrigBB.getParent();
3186  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3187  const SIRegisterInfo *TRI = ST.getRegisterInfo();
3188  MachineBasicBlock::iterator I = LoopBB.begin();
3189 
3190  const TargetRegisterClass *BoolRC = TRI->getBoolRC();
3191  Register PhiExec = MRI.createVirtualRegister(BoolRC);
3192  Register NewExec = MRI.createVirtualRegister(BoolRC);
3193  Register CurrentIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3194  Register CondReg = MRI.createVirtualRegister(BoolRC);
3195 
3196  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg)
3197  .addReg(InitReg)
3198  .addMBB(&OrigBB)
3199  .addReg(ResultReg)
3200  .addMBB(&LoopBB);
3201 
3202  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec)
3203  .addReg(InitSaveExecReg)
3204  .addMBB(&OrigBB)
3205  .addReg(NewExec)
3206  .addMBB(&LoopBB);
3207 
3208  // Read the next variant <- also loop target.
3209  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
3210  .addReg(IdxReg.getReg(), getUndefRegState(IdxReg.isUndef()));
3211 
3212  // Compare the just read M0 value to all possible Idx values.
3213  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
3214  .addReg(CurrentIdxReg)
3215  .addReg(IdxReg.getReg(), 0, IdxReg.getSubReg());
3216 
3217  // Update EXEC, save the original EXEC value to VCC.
3218  BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32
3219  : AMDGPU::S_AND_SAVEEXEC_B64),
3220  NewExec)
3221  .addReg(CondReg, RegState::Kill);
3222 
3223  MRI.setSimpleHint(NewExec, CondReg);
3224 
3225  if (UseGPRIdxMode) {
3226  unsigned IdxReg;
3227  if (Offset == 0) {
3228  IdxReg = CurrentIdxReg;
3229  } else {
3230  IdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3231  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), IdxReg)
3232  .addReg(CurrentIdxReg, RegState::Kill)
3233  .addImm(Offset);
3234  }
3235  unsigned IdxMode = IsIndirectSrc ?
3237  MachineInstr *SetOn =
3238  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
3239  .addReg(IdxReg, RegState::Kill)
3240  .addImm(IdxMode);
3241  SetOn->getOperand(3).setIsUndef();
3242  } else {
3243  // Move index from VCC into M0
3244  if (Offset == 0) {
3245  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3246  .addReg(CurrentIdxReg, RegState::Kill);
3247  } else {
3248  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
3249  .addReg(CurrentIdxReg, RegState::Kill)
3250  .addImm(Offset);
3251  }
3252  }
3253 
3254  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
3255  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
3256  MachineInstr *InsertPt =
3257  BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_XOR_B32_term
3258  : AMDGPU::S_XOR_B64_term), Exec)
3259  .addReg(Exec)
3260  .addReg(NewExec);
3261 
3262  // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
3263  // s_cbranch_scc0?
3264 
3265  // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
3266  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
3267  .addMBB(&LoopBB);
3268 
3269  return InsertPt->getIterator();
3270 }
3271 
3272 // This has slightly sub-optimal regalloc when the source vector is killed by
3273 // the read. The register allocator does not understand that the kill is
3274 // per-workitem, so is kept alive for the whole loop so we end up not re-using a
3275 // subregister from it, using 1 more VGPR than necessary. This was saved when
3276 // this was expanded after register allocation.
3278  MachineBasicBlock &MBB,
3279  MachineInstr &MI,
3280  unsigned InitResultReg,
3281  unsigned PhiReg,
3282  int Offset,
3283  bool UseGPRIdxMode,
3284  bool IsIndirectSrc) {
3285  MachineFunction *MF = MBB.getParent();
3286  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3287  const SIRegisterInfo *TRI = ST.getRegisterInfo();
3289  const DebugLoc &DL = MI.getDebugLoc();
3291 
3292  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
3293  Register DstReg = MI.getOperand(0).getReg();
3294  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
3295  Register TmpExec = MRI.createVirtualRegister(BoolXExecRC);
3296  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
3297  unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
3298 
3299  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec);
3300 
3301  // Save the EXEC mask
3302  BuildMI(MBB, I, DL, TII->get(MovExecOpc), SaveExec)
3303  .addReg(Exec);
3304 
3305  MachineBasicBlock *LoopBB;
3306  MachineBasicBlock *RemainderBB;
3307  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false);
3308 
3309  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3310 
3311  auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx,
3312  InitResultReg, DstReg, PhiReg, TmpExec,
3313  Offset, UseGPRIdxMode, IsIndirectSrc);
3314 
3315  MachineBasicBlock::iterator First = RemainderBB->begin();
3316  BuildMI(*RemainderBB, First, DL, TII->get(MovExecOpc), Exec)
3317  .addReg(SaveExec);
3318 
3319  return InsPt;
3320 }
3321 
3322 // Returns subreg index, offset
3323 static std::pair<unsigned, int>
3325  const TargetRegisterClass *SuperRC,
3326  unsigned VecReg,
3327  int Offset) {
3328  int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32;
3329 
3330  // Skip out of bounds offsets, or else we would end up using an undefined
3331  // register.
3332  if (Offset >= NumElts || Offset < 0)
3333  return std::make_pair(AMDGPU::sub0, Offset);
3334 
3335  return std::make_pair(AMDGPU::sub0 + Offset, 0);
3336 }
3337 
3338 // Return true if the index is an SGPR and was set.
3341  MachineInstr &MI,
3342  int Offset,
3343  bool UseGPRIdxMode,
3344  bool IsIndirectSrc) {
3345  MachineBasicBlock *MBB = MI.getParent();
3346  const DebugLoc &DL = MI.getDebugLoc();
3348 
3349  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3350  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
3351 
3352  assert(Idx->getReg() != AMDGPU::NoRegister);
3353 
3354  if (!TII->getRegisterInfo().isSGPRClass(IdxRC))
3355  return false;
3356 
3357  if (UseGPRIdxMode) {
3358  unsigned IdxMode = IsIndirectSrc ?
3360  if (Offset == 0) {
3361  MachineInstr *SetOn =
3362  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
3363  .add(*Idx)
3364  .addImm(IdxMode);
3365 
3366  SetOn->getOperand(3).setIsUndef();
3367  } else {
3368  Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3369  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
3370  .add(*Idx)
3371  .addImm(Offset);
3372  MachineInstr *SetOn =
3373  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
3374  .addReg(Tmp, RegState::Kill)
3375  .addImm(IdxMode);
3376 
3377  SetOn->getOperand(3).setIsUndef();
3378  }
3379 
3380  return true;
3381  }
3382 
3383  if (Offset == 0) {
3384  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3385  .add(*Idx);
3386  } else {
3387  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
3388  .add(*Idx)
3389  .addImm(Offset);
3390  }
3391 
3392  return true;
3393 }
3394 
3395 // Control flow needs to be inserted if indexing with a VGPR.
3397  MachineBasicBlock &MBB,
3398  const GCNSubtarget &ST) {
3399  const SIInstrInfo *TII = ST.getInstrInfo();
3400  const SIRegisterInfo &TRI = TII->getRegisterInfo();
3401  MachineFunction *MF = MBB.getParent();
3403 
3404  Register Dst = MI.getOperand(0).getReg();
3405  Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
3406  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
3407 
3408  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
3409 
3410  unsigned SubReg;
3411  std::tie(SubReg, Offset)
3412  = computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset);
3413 
3414  bool UseGPRIdxMode = ST.useVGPRIndexMode(EnableVGPRIndexMode);
3415 
3416  if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, true)) {
3418  const DebugLoc &DL = MI.getDebugLoc();
3419 
3420  if (UseGPRIdxMode) {
3421  // TODO: Look at the uses to avoid the copy. This may require rescheduling
3422  // to avoid interfering with other uses, so probably requires a new
3423  // optimization pass.
3424  BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
3425  .addReg(SrcReg, RegState::Undef, SubReg)
3426  .addReg(SrcReg, RegState::Implicit)
3427  .addReg(AMDGPU::M0, RegState::Implicit);
3428  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3429  } else {
3430  BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
3431  .addReg(SrcReg, RegState::Undef, SubReg)
3432  .addReg(SrcReg, RegState::Implicit);
3433  }
3434 
3435  MI.eraseFromParent();
3436 
3437  return &MBB;
3438  }
3439 
3440  const DebugLoc &DL = MI.getDebugLoc();
3442 
3443  Register PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3444  Register InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3445 
3446  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
3447 
3448  auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg,
3449  Offset, UseGPRIdxMode, true);
3450  MachineBasicBlock *LoopBB = InsPt->getParent();
3451 
3452  if (UseGPRIdxMode) {
3453  BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
3454  .addReg(SrcReg, RegState::Undef, SubReg)
3455  .addReg(SrcReg, RegState::Implicit)
3456  .addReg(AMDGPU::M0, RegState::Implicit);
3457  BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3458  } else {
3459  BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
3460  .addReg(SrcReg, RegState::Undef, SubReg)
3461  .addReg(SrcReg, RegState::Implicit);
3462  }
3463 
3464  MI.eraseFromParent();
3465 
3466  return LoopBB;
3467 }
3468 
3469 static unsigned getMOVRELDPseudo(const SIRegisterInfo &TRI,
3470  const TargetRegisterClass *VecRC) {
3471  switch (TRI.getRegSizeInBits(*VecRC)) {
3472  case 32: // 4 bytes
3473  return AMDGPU::V_MOVRELD_B32_V1;
3474  case 64: // 8 bytes
3475  return AMDGPU::V_MOVRELD_B32_V2;
3476  case 128: // 16 bytes
3477  return AMDGPU::V_MOVRELD_B32_V4;
3478  case 256: // 32 bytes
3479  return AMDGPU::V_MOVRELD_B32_V8;
3480  case 512: // 64 bytes
3481  return AMDGPU::V_MOVRELD_B32_V16;
3482  default:
3483  llvm_unreachable("unsupported size for MOVRELD pseudos");
3484  }
3485 }
3486 
3488  MachineBasicBlock &MBB,
3489  const GCNSubtarget &ST) {
3490  const SIInstrInfo *TII = ST.getInstrInfo();
3491  const SIRegisterInfo &TRI = TII->getRegisterInfo();
3492  MachineFunction *MF = MBB.getParent();
3494 
3495  Register Dst = MI.getOperand(0).getReg();
3496  const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src);
3497  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3498  const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val);
3499  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
3500  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg());
3501 
3502  // This can be an immediate, but will be folded later.
3503  assert(Val->getReg());
3504 
3505  unsigned SubReg;
3506  std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC,
3507  SrcVec->getReg(),
3508  Offset);
3509  bool UseGPRIdxMode = ST.useVGPRIndexMode(EnableVGPRIndexMode);
3510 
3511  if (Idx->getReg() == AMDGPU::NoRegister) {
3513  const DebugLoc &DL = MI.getDebugLoc();
3514 
3515  assert(Offset == 0);
3516 
3517  BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst)
3518  .add(*SrcVec)
3519  .add(*Val)
3520  .addImm(SubReg);
3521 
3522  MI.eraseFromParent();
3523  return &MBB;
3524  }
3525 
3526  if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, false)) {
3528  const DebugLoc &DL = MI.getDebugLoc();
3529 
3530  if (UseGPRIdxMode) {
3531  BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_indirect))
3532  .addReg(SrcVec->getReg(), RegState::Undef, SubReg) // vdst
3533  .add(*Val)
3534  .addReg(Dst, RegState::ImplicitDefine)
3535  .addReg(SrcVec->getReg(), RegState::Implicit)
3536  .addReg(AMDGPU::M0, RegState::Implicit);
3537 
3538  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3539  } else {
3540  const MCInstrDesc &MovRelDesc = TII->get(getMOVRELDPseudo(TRI, VecRC));
3541 
3542  BuildMI(MBB, I, DL, MovRelDesc)
3543  .addReg(Dst, RegState::Define)
3544  .addReg(SrcVec->getReg())
3545  .add(*Val)
3546  .addImm(SubReg - AMDGPU::sub0);
3547  }
3548 
3549  MI.eraseFromParent();
3550  return &MBB;
3551  }
3552 
3553  if (Val->isReg())
3554  MRI.clearKillFlags(Val->getReg());
3555 
3556  const DebugLoc &DL = MI.getDebugLoc();
3557 
3558  Register PhiReg = MRI.createVirtualRegister(VecRC);
3559 
3560  auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg,
3561  Offset, UseGPRIdxMode, false);
3562  MachineBasicBlock *LoopBB = InsPt->getParent();
3563 
3564  if (UseGPRIdxMode) {
3565  BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOV_B32_indirect))
3566  .addReg(PhiReg, RegState::Undef, SubReg) // vdst
3567  .add(*Val) // src0
3569  .addReg(PhiReg, RegState::Implicit)
3570  .addReg(AMDGPU::M0, RegState::Implicit);
3571  BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3572  } else {
3573  const MCInstrDesc &MovRelDesc = TII->get(getMOVRELDPseudo(TRI, VecRC));
3574 
3575  BuildMI(*LoopBB, InsPt, DL, MovRelDesc)
3576  .addReg(Dst, RegState::Define)
3577  .addReg(PhiReg)
3578  .add(*Val)
3579  .addImm(SubReg - AMDGPU::sub0);
3580  }
3581 
3582  MI.eraseFromParent();
3583 
3584  return LoopBB;
3585 }
3586 
3588  MachineInstr &MI, MachineBasicBlock *BB) const {
3589 
3590  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3591  MachineFunction *MF = BB->getParent();
3593 
3594  if (TII->isMIMG(MI)) {
3595  if (MI.memoperands_empty() && MI.mayLoadOrStore()) {
3596  report_fatal_error("missing mem operand from MIMG instruction");
3597  }
3598  // Add a memoperand for mimg instructions so that they aren't assumed to
3599  // be ordered memory instuctions.
3600 
3601  return BB;
3602  }
3603 
3604  switch (MI.getOpcode()) {
3605  case AMDGPU::S_ADD_U64_PSEUDO:
3606  case AMDGPU::S_SUB_U64_PSEUDO: {
3608  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3609  const SIRegisterInfo *TRI = ST.getRegisterInfo();
3610  const TargetRegisterClass *BoolRC = TRI->getBoolRC();
3611  const DebugLoc &DL = MI.getDebugLoc();
3612 
3613  MachineOperand &Dest = MI.getOperand(0);
3614  MachineOperand &Src0 = MI.getOperand(1);
3615  MachineOperand &Src1 = MI.getOperand(2);
3616 
3617  Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
3618  Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
3619 
3620  MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(MI, MRI,
3621  Src0, BoolRC, AMDGPU::sub0,
3622  &AMDGPU::SReg_32RegClass);
3623  MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(MI, MRI,
3624  Src0, BoolRC, AMDGPU::sub1,
3625  &AMDGPU::SReg_32RegClass);
3626 
3627  MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(MI, MRI,
3628  Src1, BoolRC, AMDGPU::sub0,
3629  &AMDGPU::SReg_32RegClass);
3630  MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(MI, MRI,
3631  Src1, BoolRC, AMDGPU::sub1,
3632  &AMDGPU::SReg_32RegClass);
3633 
3634  bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
3635 
3636  unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
3637  unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
3638  BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0)
3639  .add(Src0Sub0)
3640  .add(Src1Sub0);
3641  BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1)
3642  .add(Src0Sub1)
3643  .add(Src1Sub1);
3644  BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
3645  .addReg(DestSub0)
3646  .addImm(AMDGPU::sub0)
3647  .addReg(DestSub1)
3648  .addImm(AMDGPU::sub1);
3649  MI.eraseFromParent();
3650  return BB;
3651  }
3652  case AMDGPU::SI_INIT_M0: {
3653  BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
3654  TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3655  .add(MI.getOperand(0));
3656  MI.eraseFromParent();
3657  return BB;
3658  }
3659  case AMDGPU::SI_INIT_EXEC:
3660  // This should be before all vector instructions.
3661  BuildMI(*BB, &*BB->begin(), MI.getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
3662  AMDGPU::EXEC)
3663  .addImm(MI.getOperand(0).getImm());
3664  MI.eraseFromParent();
3665  return BB;
3666 
3667  case AMDGPU::SI_INIT_EXEC_LO:
3668  // This should be before all vector instructions.
3669  BuildMI(*BB, &*BB->begin(), MI.getDebugLoc(), TII->get(AMDGPU::S_MOV_B32),
3670  AMDGPU::EXEC_LO)
3671  .addImm(MI.getOperand(0).getImm());
3672  MI.eraseFromParent();
3673  return BB;
3674 
3675  case AMDGPU::SI_INIT_EXEC_FROM_INPUT: {
3676  // Extract the thread count from an SGPR input and set EXEC accordingly.
3677  // Since BFM can't shift by 64, handle that case with CMP + CMOV.
3678  //
3679  // S_BFE_U32 count, input, {shift, 7}
3680  // S_BFM_B64 exec, count, 0
3681  // S_CMP_EQ_U32 count, 64
3682  // S_CMOV_B64 exec, -1
3683  MachineInstr *FirstMI = &*BB->begin();
3685  Register InputReg = MI.getOperand(0).getReg();
3686  Register CountReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3687  bool Found = false;
3688 
3689  // Move the COPY of the input reg to the beginning, so that we can use it.
3690  for (auto I = BB->begin(); I != &MI; I++) {
3691  if (I->getOpcode() != TargetOpcode::COPY ||
3692  I->getOperand(0).getReg() != InputReg)
3693  continue;
3694 
3695  if (I == FirstMI) {
3696  FirstMI = &*++BB->begin();
3697  } else {
3698  I->removeFromParent();
3699  BB->insert(FirstMI, &*I);
3700  }
3701  Found = true;
3702  break;
3703  }
3704  assert(Found);
3705  (void)Found;
3706 
3707  // This should be before all vector instructions.
3708  unsigned Mask = (getSubtarget()->getWavefrontSize() << 1) - 1;
3709  bool isWave32 = getSubtarget()->isWave32();
3710  unsigned Exec = isWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
3711  BuildMI(*BB, FirstMI, DebugLoc(), TII->get(AMDGPU::S_BFE_U32), CountReg)
3712  .addReg(InputReg)
3713  .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000);
3714  BuildMI(*BB, FirstMI, DebugLoc(),
3715  TII->get(isWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64),
3716  Exec)
3717  .addReg(CountReg)
3718  .addImm(0);
3719  BuildMI(*BB, FirstMI, DebugLoc(), TII->get(AMDGPU::S_CMP_EQ_U32))
3720  .addReg(CountReg, RegState::Kill)
3722  BuildMI(*BB, FirstMI, DebugLoc(),
3723  TII->get(isWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
3724  Exec)
3725  .addImm(-1);
3726  MI.eraseFromParent();
3727  return BB;
3728  }
3729 
3730  case AMDGPU::GET_GROUPSTATICSIZE: {
3731  assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
3732  getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
3733  DebugLoc DL = MI.getDebugLoc();
3734  BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_MOV_B32))
3735  .add(MI.getOperand(0))
3736  .addImm(MFI->getLDSSize());
3737  MI.eraseFromParent();
3738  return BB;
3739  }
3740  case AMDGPU::SI_INDIRECT_SRC_V1:
3741  case AMDGPU::SI_INDIRECT_SRC_V2:
3742  case AMDGPU::SI_INDIRECT_SRC_V4:
3743  case AMDGPU::SI_INDIRECT_SRC_V8:
3744  case AMDGPU::SI_INDIRECT_SRC_V16:
3745  return emitIndirectSrc(MI, *BB, *getSubtarget());
3746  case AMDGPU::SI_INDIRECT_DST_V1:
3747  case AMDGPU::SI_INDIRECT_DST_V2:
3748  case AMDGPU::SI_INDIRECT_DST_V4:
3749  case AMDGPU::SI_INDIRECT_DST_V8:
3750  case AMDGPU::SI_INDIRECT_DST_V16:
3751  return emitIndirectDst(MI, *BB, *getSubtarget());
3752  case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
3753  case AMDGPU::SI_KILL_I1_PSEUDO:
3754  return splitKillBlock(MI, BB);
3755  case AMDGPU::V_CNDMASK_B64_PSEUDO: {
3757  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3758  const SIRegisterInfo *TRI = ST.getRegisterInfo();
3759 
3760  Register Dst = MI.getOperand(0).getReg();
3761  Register Src0 = MI.getOperand(1).getReg();
3762  Register Src1 = MI.getOperand(2).getReg();
3763  const DebugLoc &DL = MI.getDebugLoc();
3764  Register SrcCond = MI.getOperand(3).getReg();
3765 
3766  Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3767  Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3768  const auto *CondRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
3769  Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
3770 
3771  BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy)
3772  .addReg(SrcCond);
3773  BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
3774  .addImm(0)
3775  .addReg(Src0, 0, AMDGPU::sub0)
3776  .addImm(0)
3777  .addReg(Src1, 0, AMDGPU::sub0)
3778  .addReg(SrcCondCopy);
3779  BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
3780  .addImm(0)
3781  .addReg(Src0, 0, AMDGPU::sub1)
3782  .addImm(0)
3783  .addReg(Src1, 0, AMDGPU::sub1)
3784  .addReg(SrcCondCopy);
3785 
3786  BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
3787  .addReg(DstLo)
3788  .addImm(AMDGPU::sub0)
3789  .addReg(DstHi)
3790  .addImm(AMDGPU::sub1);
3791  MI.eraseFromParent();
3792  return BB;
3793  }
3794  case AMDGPU::SI_BR_UNDEF: {
3795  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3796  const DebugLoc &DL = MI.getDebugLoc();
3797  MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
3798  .add(MI.getOperand(0));
3799  Br->getOperand(1).setIsUndef(true); // read undef SCC
3800  MI.eraseFromParent();
3801  return BB;
3802  }
3803  case AMDGPU::ADJCALLSTACKUP:
3804  case AMDGPU::ADJCALLSTACKDOWN: {
3806  MachineInstrBuilder MIB(*MF, &MI);
3807 
3808  // Add an implicit use of the frame offset reg to prevent the restore copy
3809  // inserted after the call from being reorderd after stack operations in the
3810  // the caller's frame.
3811  MIB.addReg(Info->getStackPtrOffsetReg(), RegState::ImplicitDefine)
3812  .addReg(Info->getStackPtrOffsetReg(), RegState::Implicit)
3813  .addReg(Info->getFrameOffsetReg(), RegState::Implicit);
3814  return BB;
3815  }
3816  case AMDGPU::SI_CALL_ISEL: {
3817  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3818  const DebugLoc &DL = MI.getDebugLoc();
3819 
3820  unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF);
3821 
3822  MachineInstrBuilder MIB;
3823  MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg);
3824 
3825  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3826  MIB.add(MI.getOperand(I));
3827 
3828  MIB.cloneMemRefs(MI);
3829  MI.eraseFromParent();
3830  return BB;
3831  }
3832  case AMDGPU::V_ADD_I32_e32:
3833  case AMDGPU::V_SUB_I32_e32:
3834  case AMDGPU::V_SUBREV_I32_e32: {
3835  // TODO: Define distinct V_*_I32_Pseudo instructions instead.
3836  const DebugLoc &DL = MI.getDebugLoc();
3837  unsigned Opc = MI.getOpcode();
3838 
3839  bool NeedClampOperand = false;
3840  if (TII->pseudoToMCOpcode(Opc) == -1) {
3841  Opc = AMDGPU::getVOPe64(Opc);
3842  NeedClampOperand = true;
3843  }
3844 
3845  auto I = BuildMI(*BB, MI, DL, TII->get(Opc), MI.getOperand(0).getReg());
3846  if (TII->isVOP3(*I)) {
3847  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3848  const SIRegisterInfo *TRI = ST.getRegisterInfo();
3849  I.addReg(TRI->getVCC(), RegState::Define);
3850  }
3851  I.add(MI.getOperand(1))
3852  .add(MI.getOperand(2));
3853  if (NeedClampOperand)
3854  I.addImm(0); // clamp bit for e64 encoding
3855 
3856  TII->legalizeOperands(*I);
3857 
3858  MI.eraseFromParent();
3859  return BB;
3860  }
3861  case AMDGPU::DS_GWS_INIT:
3862  case AMDGPU::DS_GWS_SEMA_V:
3863  case AMDGPU::DS_GWS_SEMA_BR:
3864  case AMDGPU::DS_GWS_SEMA_P:
3865  case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
3866  case AMDGPU::DS_GWS_BARRIER:
3867  // A s_waitcnt 0 is required to be the instruction immediately following.
3868  if (getSubtarget()->hasGWSAutoReplay()) {
3870  return BB;
3871  }
3872 
3873  return emitGWSMemViolTestLoop(MI, BB);
3874  default:
3876  }
3877 }
3878 
3880  return isTypeLegal(VT.getScalarType());
3881 }
3882 
3884  // This currently forces unfolding various combinations of fsub into fma with
3885  // free fneg'd operands. As long as we have fast FMA (controlled by
3886  // isFMAFasterThanFMulAndFAdd), we should perform these.
3887 
3888  // When fma is quarter rate, for f64 where add / sub are at best half rate,
3889  // most of these combines appear to be cycle neutral but save on instruction
3890  // count / code size.
3891  return true;
3892 }
3893 
3895  EVT VT) const {
3896  if (!VT.isVector()) {
3897  return MVT::i1;
3898  }
3899  return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
3900 }
3901 
3903  // TODO: Should i16 be used always if legal? For now it would force VALU
3904  // shifts.
3905  return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
3906 }
3907 
3908 // Answering this is somewhat tricky and depends on the specific device which
3909 // have different rates for fma or all f64 operations.
3910 //
3911 // v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
3912 // regardless of which device (although the number of cycles differs between
3913 // devices), so it is always profitable for f64.
3914 //
3915 // v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
3916 // only on full rate devices. Normally, we should prefer selecting v_mad_f32
3917 // which we can always do even without fused FP ops since it returns the same
3918 // result as the separate operations and since it is always full
3919 // rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
3920 // however does not support denormals, so we do report fma as faster if we have
3921 // a fast fma device and require denormals.
3922 //
3924  VT = VT.getScalarType();
3925 
3926  switch (VT.getSimpleVT().SimpleTy) {
3927  case MVT::f32: {
3928  // This is as fast on some subtargets. However, we always have full rate f32
3929  // mad available which returns the same result as the separate operations
3930  // which we should prefer over fma. We can't use this if we want to support
3931  // denormals, so only report this in these cases.
3932  if (Subtarget->hasFP32Denormals())
3933  return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
3934 
3935  // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
3936  return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
3937  }
3938  case MVT::f64:
3939  return true;
3940  case MVT::f16:
3941  return Subtarget->has16BitInsts() && Subtarget->hasFP16Denormals();
3942  default:
3943  break;
3944  }
3945 
3946  return false;
3947 }
3948 
3949 //===----------------------------------------------------------------------===//
3950 // Custom DAG Lowering Operations
3951 //===----------------------------------------------------------------------===//
3952 
3953 // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
3954 // wider vector type is legal.
3956  SelectionDAG &DAG) const {
3957  unsigned Opc = Op.getOpcode();
3958  EVT VT = Op.getValueType();
3959  assert(VT == MVT::v4f16);
3960 
3961  SDValue Lo, Hi;
3962  std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
3963 
3964  SDLoc SL(Op);
3965  SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo,
3966  Op->getFlags());
3967  SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi,
3968  Op->getFlags());
3969 
3970  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
3971 }
3972 
3973 // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
3974 // wider vector type is legal.
3976  SelectionDAG &DAG) const {
3977  unsigned Opc = Op.getOpcode();
3978  EVT VT = Op.getValueType();
3979  assert(VT == MVT::v4i16 || VT == MVT::v4f16);
3980 
3981  SDValue Lo0, Hi0;
3982  std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0);
3983  SDValue Lo1, Hi1;
3984  std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1);
3985 
3986  SDLoc SL(Op);
3987 
3988  SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1,
3989  Op->getFlags());
3990  SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1,
3991  Op->getFlags());
3992 
3993  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
3994 }
3995 
3997  SelectionDAG &DAG) const {
3998  unsigned Opc = Op.getOpcode();
3999  EVT VT = Op.getValueType();
4000  assert(VT == MVT::v4i16 || VT == MVT::v4f16);
4001 
4002  SDValue Lo0, Hi0;
4003  std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0);
4004  SDValue Lo1, Hi1;
4005  std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1);
4006  SDValue Lo2, Hi2;
4007  std::tie(Lo2, Hi2) = DAG.SplitVectorOperand(Op.getNode(), 2);
4008 
4009  SDLoc SL(Op);
4010 
4011  SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Lo2,
4012  Op->getFlags());
4013  SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Hi2,
4014  Op->getFlags());
4015 
4016  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
4017 }
4018 
4019 
4021  switch (Op.getOpcode()) {
4022  default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
4023  case ISD::BRCOND: return LowerBRCOND(Op, DAG);
4024  case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
4025  case ISD::LOAD: {
4026  SDValue Result = LowerLOAD(Op, DAG);
4027  assert((!Result.getNode() ||
4028  Result.getNode()->getNumValues() == 2) &&
4029  "Load should return a value and a chain");
4030  return Result;
4031  }
4032 
4033  case ISD::FSIN:
4034  case ISD::FCOS:
4035  return LowerTrig(Op, DAG);
4036  case ISD::SELECT: return LowerSELECT(Op, DAG);
4037  case ISD::FDIV: return LowerFDIV(Op, DAG);
4038  case ISD::ATOMIC_CMP_SWAP: return LowerATOMIC_CMP_SWAP(Op, DAG);
4039  case ISD::STORE: return LowerSTORE(Op, DAG);
4040  case ISD::GlobalAddress: {
4041  MachineFunction &MF = DAG.getMachineFunction();
4043  return LowerGlobalAddress(MFI, Op, DAG);
4044  }
4045  case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4046  case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, DAG);
4047  case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
4048  case ISD::ADDRSPACECAST: return lowerADDRSPACECAST(Op, DAG);
4049  case ISD::INSERT_SUBVECTOR:
4050  return lowerINSERT_SUBVECTOR(Op, DAG);
4052  return lowerINSERT_VECTOR_ELT(Op, DAG);
4054  return lowerEXTRACT_VECTOR_ELT(Op, DAG);
4055  case ISD::VECTOR_SHUFFLE:
4056  return lowerVECTOR_SHUFFLE(Op, DAG);
4057  case ISD::BUILD_VECTOR:
4058  return lowerBUILD_VECTOR(Op, DAG);
4059  case ISD::FP_ROUND:
4060  return lowerFP_ROUND(Op, DAG);
4061  case ISD::TRAP:
4062  return lowerTRAP(Op, DAG);
4063  case ISD::DEBUGTRAP:
4064  return lowerDEBUGTRAP(Op, DAG);
4065  case ISD::FABS:
4066  case ISD::FNEG:
4067  case ISD::FCANONICALIZE:
4068  return splitUnaryVectorOp(Op, DAG);
4069  case ISD::FMINNUM:
4070  case ISD::FMAXNUM:
4071  return lowerFMINNUM_FMAXNUM(Op, DAG);
4072  case ISD::FMA:
4073  return splitTernaryVectorOp(Op, DAG);
4074  case ISD::SHL:
4075  case ISD::SRA:
4076  case ISD::SRL:
4077  case ISD::ADD:
4078  case ISD::SUB:
4079  case ISD::MUL:
4080  case ISD::SMIN:
4081  case ISD::SMAX:
4082  case ISD::UMIN:
4083  case ISD::UMAX:
4084  case ISD::FADD:
4085  case ISD::FMUL:
4086  case ISD::FMINNUM_IEEE:
4087  case ISD::FMAXNUM_IEEE:
4088  return splitBinaryVectorOp(Op, DAG);
4089  }
4090  return SDValue();
4091 }
4092 
4094  const SDLoc &DL,
4095  SelectionDAG &DAG, bool Unpacked) {
4096  if (!LoadVT.isVector())
4097  return Result;
4098 
4099  if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
4100  // Truncate to v2i16/v4i16.
4101  EVT IntLoadVT = LoadVT.changeTypeToInteger();
4102 
4103  // Workaround legalizer not scalarizing truncate after vector op
4104  // legalization byt not creating intermediate vector trunc.
4106  DAG.ExtractVectorElements(Result, Elts);
4107  for (SDValue &Elt : Elts)
4108  Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt);
4109 
4110  Result = DAG.getBuildVector(IntLoadVT, DL, Elts);
4111 
4112  // Bitcast to original type (v2f16/v4f16).
4113  return DAG.getNode(ISD::BITCAST, DL, LoadVT, Result);
4114  }
4115 
4116  // Cast back to the original packed type.
4117  return DAG.getNode(ISD::BITCAST, DL, LoadVT, Result);
4118 }
4119 
4120 SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode,
4121  MemSDNode *M,
4122  SelectionDAG &DAG,
4123  ArrayRef<SDValue> Ops,
4124  bool IsIntrinsic) const {
4125  SDLoc DL(M);
4126 
4127  bool Unpacked = Subtarget->hasUnpackedD16VMem();
4128  EVT LoadVT = M->getValueType(0);
4129 
4130  EVT EquivLoadVT = LoadVT;
4131  if (Unpacked && LoadVT.isVector()) {
4132  EquivLoadVT = LoadVT.isVector() ?
4134  LoadVT.getVectorNumElements()) : LoadVT;
4135  }
4136 
4137  // Change from v4f16/v2f16 to EquivLoadVT.
4138  SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other);
4139 
4140  SDValue Load
4141  = DAG.getMemIntrinsicNode(
4142  IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, DL,
4143  VTList, Ops, M->getMemoryVT(),
4144  M->getMemOperand());
4145  if (!Unpacked) // Just adjusted the opcode.
4146  return Load;
4147 
4148  SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked);
4149 
4150  return DAG.getMergeValues({ Adjusted, Load.getValue(1) }, DL);
4151 }
4152 
4153 SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
4154  SelectionDAG &DAG,
4155  ArrayRef<SDValue> Ops) const {
4156  SDLoc DL(M);
4157  EVT LoadVT = M->getValueType(0);
4158  EVT EltType = LoadVT.getScalarType();
4159  EVT IntVT = LoadVT.changeTypeToInteger();
4160 
4161  bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
4162 
4163  unsigned Opc =
4165 
4166  if (IsD16) {
4167  return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
4168  }
4169 
4170  // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
4171  if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
4172  return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, M);
4173 
4174  if (isTypeLegal(LoadVT)) {
4175  return getMemIntrinsicNode(Opc, DL, M->getVTList(), Ops, IntVT,
4176  M->getMemOperand(), DAG);
4177  }
4178 
4179  EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT);
4180  SDVTList VTList = DAG.getVTList(CastVT, MVT::Other);
4181  SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
4182  M->getMemOperand(), DAG);
4183  return DAG.getMergeValues(
4184  {DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode), MemNode.getValue(1)},
4185  DL);
4186 }
4187 
4189  SDNode *N, SelectionDAG &DAG) {
4190  EVT VT = N->getValueType(0);
4191  const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
4192  int CondCode = CD->getSExtValue();
4193  if (CondCode < ICmpInst::Predicate::FIRST_ICMP_PREDICATE ||
4194  CondCode > ICmpInst::Predicate::LAST_ICMP_PREDICATE)
4195  return DAG.getUNDEF(VT);
4196 
4197  ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode);
4198 
4199  SDValue LHS = N->getOperand(1);
4200  SDValue RHS = N->getOperand(2);
4201 
4202  SDLoc DL(N);
4203 
4204  EVT CmpVT = LHS.getValueType();
4205  if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) {
4206  unsigned PromoteOp = ICmpInst::isSigned(IcInput) ?
4208  LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS);
4209  RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS);
4210  }
4211 
4212  ISD::CondCode CCOpcode = getICmpCondCode(IcInput);
4213 
4214  unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
4216 
4217  SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS,
4218  DAG.getCondCode(CCOpcode));
4219  if (VT.bitsEq(CCVT))
4220  return SetCC;
4221  return DAG.getZExtOrTrunc(SetCC, DL, VT);
4222 }
4223 
4225  SDNode *N, SelectionDAG &DAG) {
4226  EVT VT = N->getValueType(0);
4227  const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
4228 
4229  int CondCode = CD->getSExtValue();
4230  if (CondCode < FCmpInst::Predicate::FIRST_FCMP_PREDICATE ||
4231  CondCode > FCmpInst::Predicate::LAST_FCMP_PREDICATE) {
4232  return DAG.getUNDEF(VT);
4233  }
4234 
4235  SDValue Src0 = N->getOperand(1);
4236  SDValue Src1 = N->getOperand(2);
4237  EVT CmpVT = Src0.getValueType();
4238  SDLoc SL(N);
4239 
4240  if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) {
4241  Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
4242  Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
4243  }
4244 
4245  FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode);
4246  ISD::CondCode CCOpcode = getFCmpCondCode(IcInput);
4247  unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
4249  SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0,
4250  Src1, DAG.getCondCode(CCOpcode));
4251  if (VT.bitsEq(CCVT))
4252  return SetCC;
4253  return DAG.getZExtOrTrunc(SetCC, SL, VT);
4254 }
4255 
4258  SelectionDAG &DAG) const {
4259  switch (N->getOpcode()) {
4260  case ISD::INSERT_VECTOR_ELT: {
4261  if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
4262  Results.push_back(Res);
4263  return;
4264  }
4265  case ISD::EXTRACT_VECTOR_ELT: {
4266  if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
4267  Results.push_back(Res);
4268  return;
4269  }
4270  case ISD::INTRINSIC_WO_CHAIN: {
4271  unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
4272  switch (IID) {
4273  case Intrinsic::amdgcn_cvt_pkrtz: {
4274  SDValue Src0 = N->getOperand(1);
4275  SDValue Src1 = N->getOperand(2);
4276  SDLoc SL(N);
4278  Src0, Src1);
4279  Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt));
4280  return;
4281  }
4282  case Intrinsic::amdgcn_cvt_pknorm_i16:
4283  case Intrinsic::amdgcn_cvt_pknorm_u16:
4284  case Intrinsic::amdgcn_cvt_pk_i16:
4285  case Intrinsic::amdgcn_cvt_pk_u16: {
4286  SDValue Src0 = N->getOperand(1);
4287  SDValue Src1 = N->getOperand(2);
4288  SDLoc SL(N);
4289  unsigned Opcode;
4290 
4291  if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
4293  else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
4295  else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
4296  Opcode = AMDGPUISD::CVT_PK_I16_I32;
4297  else
4298  Opcode = AMDGPUISD::CVT_PK_U16_U32;
4299 
4300  EVT VT = N->getValueType(0);
4301  if (isTypeLegal(VT))
4302  Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1));
4303  else {
4304  SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
4305  Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt));
4306  }
4307  return;
4308  }
4309  }
4310  break;
4311  }
4312  case ISD::INTRINSIC_W_CHAIN: {
4313  if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) {
4314  if (Res.getOpcode() == ISD::MERGE_VALUES) {
4315  // FIXME: Hacky
4316  Results.push_back(Res.getOperand(0));
4317  Results.push_back(Res.getOperand(1));
4318  } else {
4319  Results.push_back(Res);
4320  Results.push_back(Res.getValue(1));
4321  }
4322  return;
4323  }
4324 
4325  break;
4326  }
4327  case ISD::SELECT: {
4328  SDLoc SL(N);
4329  EVT VT = N->getValueType(0);
4330  EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
4331  SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
4332  SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
4333 
4334  EVT SelectVT = NewVT;
4335  if (NewVT.bitsLT(MVT::i32)) {
4336  LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS);
4337  RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS);
4338  SelectVT = MVT::i32;
4339  }
4340 
4341  SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT,
4342  N->getOperand(0), LHS, RHS);
4343 
4344  if (NewVT != SelectVT)
4345  NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect);
4346  Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect));
4347  return;
4348  }
4349  case ISD::FNEG: {
4350  if (N->getValueType(0) != MVT::v2f16)
4351  break;
4352 
4353  SDLoc SL(N);
4354  SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4355 
4356  SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32,
4357  BC,
4358  DAG.getConstant(0x80008000, SL, MVT::i32));
4359  Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
4360  return;
4361  }
4362  case ISD::FABS: {
4363  if (N->getValueType(0) != MVT::v2f16)
4364  break;
4365 
4366  SDLoc SL(N);
4367  SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4368 
4369  SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32,
4370  BC,
4371  DAG.getConstant(0x7fff7fff, SL, MVT::i32));
4372  Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
4373  return;
4374  }
4375  default:
4376  break;
4377  }
4378 }
4379 
4380 /// Helper function for LowerBRCOND