LLVM  6.0.0svn
AMDGPUISelLowering.cpp
Go to the documentation of this file.
1 //===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief This is the parent TargetLowering class for hardware code gen
12 /// targets.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "AMDGPUISelLowering.h"
17 #include "AMDGPU.h"
18 #include "AMDGPUCallLowering.h"
19 #include "AMDGPUFrameLowering.h"
20 #include "AMDGPUIntrinsicInfo.h"
21 #include "AMDGPURegisterInfo.h"
22 #include "AMDGPUSubtarget.h"
23 #include "AMDGPUTargetMachine.h"
25 #include "SIInstrInfo.h"
26 #include "SIMachineFunctionInfo.h"
32 #include "llvm/IR/DataLayout.h"
33 #include "llvm/IR/DiagnosticInfo.h"
34 #include "llvm/Support/KnownBits.h"
35 using namespace llvm;
36 
37 static bool allocateKernArg(unsigned ValNo, MVT ValVT, MVT LocVT,
38  CCValAssign::LocInfo LocInfo,
39  ISD::ArgFlagsTy ArgFlags, CCState &State) {
40  MachineFunction &MF = State.getMachineFunction();
42 
43  uint64_t Offset = MFI->allocateKernArg(LocVT.getStoreSize(),
44  ArgFlags.getOrigAlign());
45  State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo));
46  return true;
47 }
48 
49 static bool allocateCCRegs(unsigned ValNo, MVT ValVT, MVT LocVT,
50  CCValAssign::LocInfo LocInfo,
51  ISD::ArgFlagsTy ArgFlags, CCState &State,
52  const TargetRegisterClass *RC,
53  unsigned NumRegs) {
54  ArrayRef<MCPhysReg> RegList = makeArrayRef(RC->begin(), NumRegs);
55  unsigned RegResult = State.AllocateReg(RegList);
56  if (RegResult == AMDGPU::NoRegister)
57  return false;
58 
59  State.addLoc(CCValAssign::getReg(ValNo, ValVT, RegResult, LocVT, LocInfo));
60  return true;
61 }
62 
63 static bool allocateSGPRTuple(unsigned ValNo, MVT ValVT, MVT LocVT,
64  CCValAssign::LocInfo LocInfo,
65  ISD::ArgFlagsTy ArgFlags, CCState &State) {
66  switch (LocVT.SimpleTy) {
67  case MVT::i64:
68  case MVT::f64:
69  case MVT::v2i32:
70  case MVT::v2f32: {
71  // Up to SGPR0-SGPR39
72  return allocateCCRegs(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State,
73  &AMDGPU::SGPR_64RegClass, 20);
74  }
75  default:
76  return false;
77  }
78 }
79 
80 // Allocate up to VGPR31.
81 //
82 // TODO: Since there are no VGPR alignent requirements would it be better to
83 // split into individual scalar registers?
84 static bool allocateVGPRTuple(unsigned ValNo, MVT ValVT, MVT LocVT,
85  CCValAssign::LocInfo LocInfo,
86  ISD::ArgFlagsTy ArgFlags, CCState &State) {
87  switch (LocVT.SimpleTy) {
88  case MVT::i64:
89  case MVT::f64:
90  case MVT::v2i32:
91  case MVT::v2f32: {
92  return allocateCCRegs(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State,
93  &AMDGPU::VReg_64RegClass, 31);
94  }
95  case MVT::v4i32:
96  case MVT::v4f32:
97  case MVT::v2i64:
98  case MVT::v2f64: {
99  return allocateCCRegs(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State,
100  &AMDGPU::VReg_128RegClass, 29);
101  }
102  case MVT::v8i32:
103  case MVT::v8f32: {
104  return allocateCCRegs(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State,
105  &AMDGPU::VReg_256RegClass, 25);
106 
107  }
108  case MVT::v16i32:
109  case MVT::v16f32: {
110  return allocateCCRegs(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State,
111  &AMDGPU::VReg_512RegClass, 17);
112 
113  }
114  default:
115  return false;
116  }
117 }
118 
119 #include "AMDGPUGenCallingConv.inc"
120 
121 // Find a larger type to do a load / store of a vector with.
123  unsigned StoreSize = VT.getStoreSizeInBits();
124  if (StoreSize <= 32)
125  return EVT::getIntegerVT(Ctx, StoreSize);
126 
127  assert(StoreSize % 32 == 0 && "Store size not a multiple of 32");
128  return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32);
129 }
130 
132  KnownBits Known;
133  EVT VT = Op.getValueType();
134  DAG.computeKnownBits(Op, Known);
135 
136  return VT.getSizeInBits() - Known.countMinLeadingZeros();
137 }
138 
140  EVT VT = Op.getValueType();
141 
142  // In order for this to be a signed 24-bit value, bit 23, must
143  // be a sign bit.
144  return VT.getSizeInBits() - DAG.ComputeNumSignBits(Op);
145 }
146 
148  const AMDGPUSubtarget &STI)
149  : TargetLowering(TM), Subtarget(&STI) {
151  // Lower floating point store/load to integer store/load to reduce the number
152  // of patterns in tablegen.
155 
158 
161 
164 
167 
170 
173 
176 
179 
180  // There are no 64-bit extloads. These should be done as a 32-bit extload and
181  // an extension to 64-bit.
182  for (MVT VT : MVT::integer_valuetypes()) {
186  }
187 
188  for (MVT VT : MVT::integer_valuetypes()) {
189  if (VT == MVT::i64)
190  continue;
191 
196 
201 
206  }
207 
208  for (MVT VT : MVT::integer_vector_valuetypes()) {
221  }
222 
227 
232 
237 
240 
243 
246 
249 
252 
255 
258 
261 
264 
269 
274 
279 
282 
285 
288 
291 
292 
297 
300 
301  // This is totally unsupported, just custom lower to produce an error.
303 
304  // Library functions. These default to Expand, but we have instructions
305  // for them.
316 
319 
322 
325 
326  // v_mad_f32 does not support denormals according to some sources.
327  if (!Subtarget->hasFP32Denormals())
329 
330  // Expand to fneg + fadd.
332 
343 
349  }
350 
351  if (!Subtarget->hasBFI()) {
352  // fcopysign can be done in a single instruction with BFI.
355  }
356 
360 
361  const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
362  for (MVT VT : ScalarIntVTs) {
363  // These should use [SU]DIVREM, so set them to expand
368 
369  // GPU does not have divrem function for signed or unsigned.
372 
373  // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
376 
380  }
381 
382  if (!Subtarget->hasBCNT(32))
384 
385  if (!Subtarget->hasBCNT(64))
387 
388  // The hardware supports 32-bit ROTR, but not ROTL.
390  setOperationAction(ISD::ROTL, MVT::i64, Expand);
391  setOperationAction(ISD::ROTR, MVT::i64, Expand);
392 
393  setOperationAction(ISD::MUL, MVT::i64, Expand);
401 
406 
407  if (Subtarget->hasFFBH())
409 
410  if (Subtarget->hasFFBL())
412 
413  setOperationAction(ISD::CTTZ, MVT::i64, Custom);
415  setOperationAction(ISD::CTLZ, MVT::i64, Custom);
417 
418  // We only really have 32-bit BFE instructions (and 16-bit on VI).
419  //
420  // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
421  // effort to match them now. We want this to be false for i64 cases when the
422  // extraction isn't restricted to the upper or lower half. Ideally we would
423  // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
424  // span the midpoint are probably relatively rare, so don't worry about them
425  // for now.
426  if (Subtarget->hasBFE())
427  setHasExtractBitsInsn(true);
428 
429  static const MVT::SimpleValueType VectorIntTypes[] = {
431  };
432 
433  for (MVT VT : VectorIntTypes) {
434  // Expand the following operations for the current type by default.
473  }
474 
475  static const MVT::SimpleValueType FloatVectorTypes[] = {
477  };
478 
479  for (MVT VT : FloatVectorTypes) {
506  }
507 
508  // This causes using an unrolled select operation rather than expansion with
509  // bit operations. This is in general better, but the alternative using BFI
510  // instructions may be better if the select sources are SGPRs.
513 
514  setOperationAction(ISD::SELECT, MVT::v4f32, Promote);
515  AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32);
516 
517  // There are no libcalls of any kind.
518  for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
519  setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
520 
523 
525  setJumpIsExpensive(true);
526 
527  // FIXME: This is only partially true. If we have to do vector compares, any
528  // SGPR pair can be a condition register. If we have a uniform condition, we
529  // are better off doing SALU operations, where there is only one SCC. For now,
530  // we don't have a way of knowing during instruction selection if a condition
531  // will be uniform and we always use vector compares. Assume we are using
532  // vector compares until that is fixed.
534 
535  // SI at least has hardware support for floating point exceptions, but no way
536  // of using or handling them is implemented. They are also optional in OpenCL
537  // (Section 7.3)
539 
541 
542  // We want to find all load dependencies for long chains of stores to enable
543  // merging into very wide vectors. The problem is with vectors with > 4
544  // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
545  // vectors are a legal type, even though we have to split the loads
546  // usually. When we can more precisely specify load legality per address
547  // space, we should be able to make FindBetterChain/MergeConsecutiveStores
548  // smarter so that they can figure out what to do in 2 iterations without all
549  // N > 4 stores on the same chain.
551 
552  // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
553  // about these during lowering.
554  MaxStoresPerMemcpy = 0xffffffff;
555  MaxStoresPerMemmove = 0xffffffff;
556  MaxStoresPerMemset = 0xffffffff;
557 
574 }
575 
576 //===----------------------------------------------------------------------===//
577 // Target Information
578 //===----------------------------------------------------------------------===//
579 
581 static bool fnegFoldsIntoOp(unsigned Opc) {
582  switch (Opc) {
583  case ISD::FADD:
584  case ISD::FSUB:
585  case ISD::FMUL:
586  case ISD::FMA:
587  case ISD::FMAD:
588  case ISD::FMINNUM:
589  case ISD::FMAXNUM:
590  case ISD::FSIN:
591  case ISD::FTRUNC:
592  case ISD::FRINT:
593  case ISD::FNEARBYINT:
594  case AMDGPUISD::RCP:
596  case AMDGPUISD::SIN_HW:
600  return true;
601  default:
602  return false;
603  }
604 }
605 
606 /// \p returns true if the operation will definitely need to use a 64-bit
607 /// encoding, and thus will use a VOP3 encoding regardless of the source
608 /// modifiers.
610 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
611  return N->getNumOperands() > 2 || VT == MVT::f64;
612 }
613 
614 // Most FP instructions support source modifiers, but this could be refined
615 // slightly.
617 static bool hasSourceMods(const SDNode *N) {
618  if (isa<MemSDNode>(N))
619  return false;
620 
621  switch (N->getOpcode()) {
622  case ISD::CopyToReg:
623  case ISD::SELECT:
624  case ISD::FDIV:
625  case ISD::FREM:
626  case ISD::INLINEASM:
630 
631  // TODO: Should really be looking at the users of the bitcast. These are
632  // problematic because bitcasts are used to legalize all stores to integer
633  // types.
634  case ISD::BITCAST:
635  return false;
636  default:
637  return true;
638  }
639 }
640 
642  unsigned CostThreshold) {
643  // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
644  // it is truly free to use a source modifier in all cases. If there are
645  // multiple users but for each one will necessitate using VOP3, there will be
646  // a code size increase. Try to avoid increasing code size unless we know it
647  // will save on the instruction count.
648  unsigned NumMayIncreaseSize = 0;
649  MVT VT = N->getValueType(0).getScalarType().getSimpleVT();
650 
651  // XXX - Should this limit number of uses to check?
652  for (const SDNode *U : N->uses()) {
653  if (!hasSourceMods(U))
654  return false;
655 
656  if (!opMustUseVOP3Encoding(U, VT)) {
657  if (++NumMayIncreaseSize > CostThreshold)
658  return false;
659  }
660  }
661 
662  return true;
663 }
664 
666  return MVT::i32;
667 }
668 
670  return true;
671 }
672 
673 // The backend supports 32 and 64 bit floating point immediates.
674 // FIXME: Why are we reporting vectors of FP immediates as legal?
675 bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
676  EVT ScalarVT = VT.getScalarType();
677  return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 ||
678  (ScalarVT == MVT::f16 && Subtarget->has16BitInsts()));
679 }
680 
681 // We don't want to shrink f64 / f32 constants.
683  EVT ScalarVT = VT.getScalarType();
684  return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
685 }
686 
689  EVT NewVT) const {
690 
691  unsigned NewSize = NewVT.getStoreSizeInBits();
692 
693  // If we are reducing to a 32-bit load, this is always better.
694  if (NewSize == 32)
695  return true;
696 
697  EVT OldVT = N->getValueType(0);
698  unsigned OldSize = OldVT.getStoreSizeInBits();
699 
700  // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
701  // extloads, so doing one requires using a buffer_load. In cases where we
702  // still couldn't use a scalar load, using the wider load shouldn't really
703  // hurt anything.
704 
705  // If the old size already had to be an extload, there's no harm in continuing
706  // to reduce the width.
707  return (OldSize < 32);
708 }
709 
711  EVT CastTy) const {
712 
713  assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
714 
715  if (LoadTy.getScalarType() == MVT::i32)
716  return false;
717 
718  unsigned LScalarSize = LoadTy.getScalarSizeInBits();
719  unsigned CastScalarSize = CastTy.getScalarSizeInBits();
720 
721  return (LScalarSize < CastScalarSize) ||
722  (CastScalarSize >= 32);
723 }
724 
725 // SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
726 // profitable with the expansion for 64-bit since it's generally good to
727 // speculate things.
728 // FIXME: These should really have the size as a parameter.
730  return true;
731 }
732 
734  return true;
735 }
736 
737 //===---------------------------------------------------------------------===//
738 // Target Properties
739 //===---------------------------------------------------------------------===//
740 
742  assert(VT.isFloatingPoint());
743 
744  // Packed operations do not have a fabs modifier.
745  return VT == MVT::f32 || VT == MVT::f64 ||
746  (Subtarget->has16BitInsts() && VT == MVT::f16);
747 }
748 
750  assert(VT.isFloatingPoint());
751  return VT == MVT::f32 || VT == MVT::f64 ||
752  (Subtarget->has16BitInsts() && VT == MVT::f16) ||
753  (Subtarget->hasVOP3PInsts() && VT == MVT::v2f16);
754 }
755 
757  unsigned NumElem,
758  unsigned AS) const {
759  return true;
760 }
761 
763  // There are few operations which truly have vector input operands. Any vector
764  // operation is going to involve operations on each component, and a
765  // build_vector will be a copy per element, so it always makes sense to use a
766  // build_vector input in place of the extracted element to avoid a copy into a
767  // super register.
768  //
769  // We should probably only do this if all users are extracts only, but this
770  // should be the common case.
771  return true;
772 }
773 
775  // Truncate is just accessing a subregister.
776 
777  unsigned SrcSize = Source.getSizeInBits();
778  unsigned DestSize = Dest.getSizeInBits();
779 
780  return DestSize < SrcSize && DestSize % 32 == 0 ;
781 }
782 
784  // Truncate is just accessing a subregister.
785 
786  unsigned SrcSize = Source->getScalarSizeInBits();
787  unsigned DestSize = Dest->getScalarSizeInBits();
788 
789  if (DestSize== 16 && Subtarget->has16BitInsts())
790  return SrcSize >= 32;
791 
792  return DestSize < SrcSize && DestSize % 32 == 0;
793 }
794 
795 bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
796  unsigned SrcSize = Src->getScalarSizeInBits();
797  unsigned DestSize = Dest->getScalarSizeInBits();
798 
799  if (SrcSize == 16 && Subtarget->has16BitInsts())
800  return DestSize >= 32;
801 
802  return SrcSize == 32 && DestSize == 64;
803 }
804 
805 bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
806  // Any register load of a 64-bit value really requires 2 32-bit moves. For all
807  // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
808  // this will enable reducing 64-bit operations the 32-bit, which is always
809  // good.
810 
811  if (Src == MVT::i16)
812  return Dest == MVT::i32 ||Dest == MVT::i64 ;
813 
814  return Src == MVT::i32 && Dest == MVT::i64;
815 }
816 
818  return isZExtFree(Val.getValueType(), VT2);
819 }
820 
821 // v_mad_mix* support a conversion from f16 to f32.
822 //
823 // There is only one special case when denormals are enabled we don't currently,
824 // where this is OK to use.
826  EVT DestVT, EVT SrcVT) const {
827  return Opcode == ISD::FMAD && Subtarget->hasMadMixInsts() &&
828  DestVT.getScalarType() == MVT::f32 && !Subtarget->hasFP32Denormals() &&
829  SrcVT.getScalarType() == MVT::f16;
830 }
831 
833  // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
834  // limited number of native 64-bit operations. Shrinking an operation to fit
835  // in a single 32-bit register should always be helpful. As currently used,
836  // this is much less general than the name suggests, and is only used in
837  // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
838  // not profitable, and may actually be harmful.
839  return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
840 }
841 
842 //===---------------------------------------------------------------------===//
843 // TargetLowering Callbacks
844 //===---------------------------------------------------------------------===//
845 
847  bool IsVarArg) {
848  switch (CC) {
851  return CC_AMDGPU_Kernel;
859  return CC_AMDGPU;
860  case CallingConv::C:
861  case CallingConv::Fast:
862  case CallingConv::Cold:
863  return CC_AMDGPU_Func;
864  default:
865  report_fatal_error("Unsupported calling convention.");
866  }
867 }
868 
870  bool IsVarArg) {
871  switch (CC) {
874  return CC_AMDGPU_Kernel;
882  return RetCC_SI_Shader;
883  case CallingConv::C:
884  case CallingConv::Fast:
885  case CallingConv::Cold:
886  return RetCC_AMDGPU_Func;
887  default:
888  report_fatal_error("Unsupported calling convention.");
889  }
890 }
891 
892 /// The SelectionDAGBuilder will automatically promote function arguments
893 /// with illegal types. However, this does not work for the AMDGPU targets
894 /// since the function arguments are stored in memory as these illegal types.
895 /// In order to handle this properly we need to get the original types sizes
896 /// from the LLVM IR Function and fixup the ISD:InputArg values before
897 /// passing them to AnalyzeFormalArguments()
898 
899 /// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
900 /// input values across multiple registers. Each item in the Ins array
901 /// represents a single value that will be stored in registers. Ins[x].VT is
902 /// the value type of the value that will be stored in the register, so
903 /// whatever SDNode we lower the argument to needs to be this type.
904 ///
905 /// In order to correctly lower the arguments we need to know the size of each
906 /// argument. Since Ins[x].VT gives us the size of the register that will
907 /// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
908 /// for the orignal function argument so that we can deduce the correct memory
909 /// type to use for Ins[x]. In most cases the correct memory type will be
910 /// Ins[x].ArgVT. However, this will not always be the case. If, for example,
911 /// we have a kernel argument of type v8i8, this argument will be split into
912 /// 8 parts and each part will be represented by its own item in the Ins array.
913 /// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
914 /// the argument before it was split. From this, we deduce that the memory type
915 /// for each individual part is i8. We pass the memory type as LocVT to the
916 /// calling convention analysis function and the register type (Ins[x].VT) as
917 /// the ValVT.
919  const SmallVectorImpl<ISD::InputArg> &Ins) const {
920  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
921  const ISD::InputArg &In = Ins[i];
922  EVT MemVT;
923 
924  unsigned NumRegs = getNumRegisters(State.getContext(), In.ArgVT);
925 
926  if (!Subtarget->isAmdHsaOS() &&
927  (In.ArgVT == MVT::i16 || In.ArgVT == MVT::i8 || In.ArgVT == MVT::f16)) {
928  // The ABI says the caller will extend these values to 32-bits.
929  MemVT = In.ArgVT.isInteger() ? MVT::i32 : MVT::f32;
930  } else if (NumRegs == 1) {
931  // This argument is not split, so the IR type is the memory type.
932  assert(!In.Flags.isSplit());
933  if (In.ArgVT.isExtended()) {
934  // We have an extended type, like i24, so we should just use the register type
935  MemVT = In.VT;
936  } else {
937  MemVT = In.ArgVT;
938  }
939  } else if (In.ArgVT.isVector() && In.VT.isVector() &&
940  In.ArgVT.getScalarType() == In.VT.getScalarType()) {
942  // We have a vector value which has been split into a vector with
943  // the same scalar type, but fewer elements. This should handle
944  // all the floating-point vector types.
945  MemVT = In.VT;
946  } else if (In.ArgVT.isVector() &&
947  In.ArgVT.getVectorNumElements() == NumRegs) {
948  // This arg has been split so that each element is stored in a separate
949  // register.
950  MemVT = In.ArgVT.getScalarType();
951  } else if (In.ArgVT.isExtended()) {
952  // We have an extended type, like i65.
953  MemVT = In.VT;
954  } else {
955  unsigned MemoryBits = In.ArgVT.getStoreSizeInBits() / NumRegs;
956  assert(In.ArgVT.getStoreSizeInBits() % NumRegs == 0);
957  if (In.VT.isInteger()) {
958  MemVT = EVT::getIntegerVT(State.getContext(), MemoryBits);
959  } else if (In.VT.isVector()) {
961  unsigned NumElements = In.VT.getVectorNumElements();
962  assert(MemoryBits % NumElements == 0);
963  // This vector type has been split into another vector type with
964  // a different elements size.
965  EVT ScalarVT = EVT::getIntegerVT(State.getContext(),
966  MemoryBits / NumElements);
967  MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements);
968  } else {
969  llvm_unreachable("cannot deduce memory type.");
970  }
971  }
972 
973  // Convert one element vectors to scalar.
974  if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
975  MemVT = MemVT.getScalarType();
976 
977  if (MemVT.isExtended()) {
978  // This should really only happen if we have vec3 arguments
979  assert(MemVT.isVector() && MemVT.getVectorNumElements() == 3);
980  MemVT = MemVT.getPow2VectorType(State.getContext());
981  }
982 
983  assert(MemVT.isSimple());
985  State);
986  }
987 }
988 
990  SDValue Chain, CallingConv::ID CallConv,
991  bool isVarArg,
993  const SmallVectorImpl<SDValue> &OutVals,
994  const SDLoc &DL, SelectionDAG &DAG) const {
995  // FIXME: Fails for r600 tests
996  //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
997  // "wave terminate should not have return values");
998  return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain);
999 }
1000 
1001 //===---------------------------------------------------------------------===//
1002 // Target specific lowering
1003 //===---------------------------------------------------------------------===//
1004 
1005 /// Selects the correct CCAssignFn for a given CallingConvention value.
1007  bool IsVarArg) {
1008  return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1009 }
1010 
1012  bool IsVarArg) {
1013  return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1014 }
1015 
1017  SelectionDAG &DAG,
1018  MachineFrameInfo &MFI,
1019  int ClobberedFI) const {
1020  SmallVector<SDValue, 8> ArgChains;
1021  int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
1022  int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
1023 
1024  // Include the original chain at the beginning of the list. When this is
1025  // used by target LowerCall hooks, this helps legalize find the
1026  // CALLSEQ_BEGIN node.
1027  ArgChains.push_back(Chain);
1028 
1029  // Add a chain value for each stack argument corresponding
1030  for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
1031  UE = DAG.getEntryNode().getNode()->use_end();
1032  U != UE; ++U) {
1033  if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) {
1034  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) {
1035  if (FI->getIndex() < 0) {
1036  int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
1037  int64_t InLastByte = InFirstByte;
1038  InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
1039 
1040  if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1041  (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1042  ArgChains.push_back(SDValue(L, 1));
1043  }
1044  }
1045  }
1046  }
1047 
1048  // Build a tokenfactor for all the chains.
1049  return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
1050 }
1051 
1053  SmallVectorImpl<SDValue> &InVals,
1054  StringRef Reason) const {
1055  SDValue Callee = CLI.Callee;
1056  SelectionDAG &DAG = CLI.DAG;
1057 
1058  const Function &Fn = *DAG.getMachineFunction().getFunction();
1059 
1060  StringRef FuncName("<unknown>");
1061 
1062  if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee))
1063  FuncName = G->getSymbol();
1064  else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
1065  FuncName = G->getGlobal()->getName();
1066 
1067  DiagnosticInfoUnsupported NoCalls(
1068  Fn, Reason + FuncName, CLI.DL.getDebugLoc());
1069  DAG.getContext()->diagnose(NoCalls);
1070 
1071  if (!CLI.IsTailCall) {
1072  for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I)
1073  InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT));
1074  }
1075 
1076  return DAG.getEntryNode();
1077 }
1078 
1080  SmallVectorImpl<SDValue> &InVals) const {
1081  return lowerUnhandledCall(CLI, InVals, "unsupported call to function ");
1082 }
1083 
1085  SelectionDAG &DAG) const {
1086  const Function &Fn = *DAG.getMachineFunction().getFunction();
1087 
1088  DiagnosticInfoUnsupported NoDynamicAlloca(Fn, "unsupported dynamic alloca",
1089  SDLoc(Op).getDebugLoc());
1090  DAG.getContext()->diagnose(NoDynamicAlloca);
1091  auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
1092  return DAG.getMergeValues(Ops, SDLoc());
1093 }
1094 
1096  SelectionDAG &DAG) const {
1097  switch (Op.getOpcode()) {
1098  default:
1099  Op->print(errs(), &DAG);
1100  llvm_unreachable("Custom lowering code for this"
1101  "instruction is not implemented yet!");
1102  break;
1103  case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1104  case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1105  case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1106  case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1107  case ISD::SDIVREM: return LowerSDIVREM(Op, DAG);
1108  case ISD::FREM: return LowerFREM(Op, DAG);
1109  case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1110  case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1111  case ISD::FRINT: return LowerFRINT(Op, DAG);
1112  case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1113  case ISD::FROUND: return LowerFROUND(Op, DAG);
1114  case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1115  case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1116  case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1117  case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1118  case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
1119  case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
1120  case ISD::CTTZ:
1121  case ISD::CTTZ_ZERO_UNDEF:
1122  case ISD::CTLZ:
1123  case ISD::CTLZ_ZERO_UNDEF:
1124  return LowerCTLZ_CTTZ(Op, DAG);
1125  case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1126  }
1127  return Op;
1128 }
1129 
1132  SelectionDAG &DAG) const {
1133  switch (N->getOpcode()) {
1135  // Different parts of legalization seem to interpret which type of
1136  // sign_extend_inreg is the one to check for custom lowering. The extended
1137  // from type is what really matters, but some places check for custom
1138  // lowering of the result type. This results in trying to use
1139  // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1140  // nothing here and let the illegal result integer be handled normally.
1141  return;
1142  default:
1143  return;
1144  }
1145 }
1146 
1147 static bool hasDefinedInitializer(const GlobalValue *GV) {
1148  const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
1149  if (!GVar || !GVar->hasInitializer())
1150  return false;
1151 
1152  return !isa<UndefValue>(GVar->getInitializer());
1153 }
1154 
1156  SDValue Op,
1157  SelectionDAG &DAG) const {
1158 
1159  const DataLayout &DL = DAG.getDataLayout();
1160  GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
1161  const GlobalValue *GV = G->getGlobal();
1162 
1163  if (G->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS) {
1164  // XXX: What does the value of G->getOffset() mean?
1165  assert(G->getOffset() == 0 &&
1166  "Do not know what to do with an non-zero offset");
1167 
1168  // TODO: We could emit code to handle the initialization somewhere.
1169  if (!hasDefinedInitializer(GV)) {
1170  unsigned Offset = MFI->allocateLDSGlobal(DL, *GV);
1171  return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
1172  }
1173  }
1174 
1175  const Function &Fn = *DAG.getMachineFunction().getFunction();
1176  DiagnosticInfoUnsupported BadInit(
1177  Fn, "unsupported initializer for address space", SDLoc(Op).getDebugLoc());
1178  DAG.getContext()->diagnose(BadInit);
1179  return SDValue();
1180 }
1181 
1183  SelectionDAG &DAG) const {
1185 
1186  for (const SDUse &U : Op->ops())
1187  DAG.ExtractVectorElements(U.get(), Args);
1188 
1189  return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
1190 }
1191 
1193  SelectionDAG &DAG) const {
1194 
1196  unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
1197  EVT VT = Op.getValueType();
1198  DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
1199  VT.getVectorNumElements());
1200 
1201  return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
1202 }
1203 
1204 /// \brief Generate Min/Max node
1206  SDValue LHS, SDValue RHS,
1207  SDValue True, SDValue False,
1208  SDValue CC,
1209  DAGCombinerInfo &DCI) const {
1210  if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
1211  return SDValue();
1212 
1213  SelectionDAG &DAG = DCI.DAG;
1214  ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
1215  switch (CCOpcode) {
1216  case ISD::SETOEQ:
1217  case ISD::SETONE:
1218  case ISD::SETUNE:
1219  case ISD::SETNE:
1220  case ISD::SETUEQ:
1221  case ISD::SETEQ:
1222  case ISD::SETFALSE:
1223  case ISD::SETFALSE2:
1224  case ISD::SETTRUE:
1225  case ISD::SETTRUE2:
1226  case ISD::SETUO:
1227  case ISD::SETO:
1228  break;
1229  case ISD::SETULE:
1230  case ISD::SETULT: {
1231  if (LHS == True)
1232  return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1233  return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1234  }
1235  case ISD::SETOLE:
1236  case ISD::SETOLT:
1237  case ISD::SETLE:
1238  case ISD::SETLT: {
1239  // Ordered. Assume ordered for undefined.
1240 
1241  // Only do this after legalization to avoid interfering with other combines
1242  // which might occur.
1243  if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1244  !DCI.isCalledByLegalizer())
1245  return SDValue();
1246 
1247  // We need to permute the operands to get the correct NaN behavior. The
1248  // selected operand is the second one based on the failing compare with NaN,
1249  // so permute it based on the compare type the hardware uses.
1250  if (LHS == True)
1251  return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1252  return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1253  }
1254  case ISD::SETUGE:
1255  case ISD::SETUGT: {
1256  if (LHS == True)
1257  return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1258  return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1259  }
1260  case ISD::SETGT:
1261  case ISD::SETGE:
1262  case ISD::SETOGE:
1263  case ISD::SETOGT: {
1264  if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1265  !DCI.isCalledByLegalizer())
1266  return SDValue();
1267 
1268  if (LHS == True)
1269  return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1270  return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1271  }
1272  case ISD::SETCC_INVALID:
1273  llvm_unreachable("Invalid setcc condcode!");
1274  }
1275  return SDValue();
1276 }
1277 
1278 std::pair<SDValue, SDValue>
1280  SDLoc SL(Op);
1281 
1282  SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1283 
1284  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1285  const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1286 
1287  SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1288  SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1289 
1290  return std::make_pair(Lo, Hi);
1291 }
1292 
1294  SDLoc SL(Op);
1295 
1296  SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1297  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1298  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1299 }
1300 
1302  SDLoc SL(Op);
1303 
1304  SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1305  const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1306  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1307 }
1308 
1310  SelectionDAG &DAG) const {
1311  LoadSDNode *Load = cast<LoadSDNode>(Op);
1312  EVT VT = Op.getValueType();
1313 
1314 
1315  // If this is a 2 element vector, we really want to scalarize and not create
1316  // weird 1 element vectors.
1317  if (VT.getVectorNumElements() == 2)
1318  return scalarizeVectorLoad(Load, DAG);
1319 
1320  SDValue BasePtr = Load->getBasePtr();
1321  EVT PtrVT = BasePtr.getValueType();
1322  EVT MemVT = Load->getMemoryVT();
1323  SDLoc SL(Op);
1324 
1325  const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1326 
1327  EVT LoVT, HiVT;
1328  EVT LoMemVT, HiMemVT;
1329  SDValue Lo, Hi;
1330 
1331  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
1332  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT);
1333  std::tie(Lo, Hi) = DAG.SplitVector(Op, SL, LoVT, HiVT);
1334 
1335  unsigned Size = LoMemVT.getStoreSize();
1336  unsigned BaseAlign = Load->getAlignment();
1337  unsigned HiAlign = MinAlign(BaseAlign, Size);
1338 
1339  SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
1340  Load->getChain(), BasePtr, SrcValue, LoMemVT,
1341  BaseAlign, Load->getMemOperand()->getFlags());
1342  SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
1343  DAG.getConstant(Size, SL, PtrVT));
1344  SDValue HiLoad =
1345  DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(),
1346  HiPtr, SrcValue.getWithOffset(LoMemVT.getStoreSize()),
1347  HiMemVT, HiAlign, Load->getMemOperand()->getFlags());
1348 
1349  SDValue Ops[] = {
1350  DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad),
1352  LoLoad.getValue(1), HiLoad.getValue(1))
1353  };
1354 
1355  return DAG.getMergeValues(Ops, SL);
1356 }
1357 
1359  SelectionDAG &DAG) const {
1360  StoreSDNode *Store = cast<StoreSDNode>(Op);
1361  SDValue Val = Store->getValue();
1362  EVT VT = Val.getValueType();
1363 
1364  // If this is a 2 element vector, we really want to scalarize and not create
1365  // weird 1 element vectors.
1366  if (VT.getVectorNumElements() == 2)
1367  return scalarizeVectorStore(Store, DAG);
1368 
1369  EVT MemVT = Store->getMemoryVT();
1370  SDValue Chain = Store->getChain();
1371  SDValue BasePtr = Store->getBasePtr();
1372  SDLoc SL(Op);
1373 
1374  EVT LoVT, HiVT;
1375  EVT LoMemVT, HiMemVT;
1376  SDValue Lo, Hi;
1377 
1378  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
1379  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT);
1380  std::tie(Lo, Hi) = DAG.SplitVector(Val, SL, LoVT, HiVT);
1381 
1382  EVT PtrVT = BasePtr.getValueType();
1383  SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
1384  DAG.getConstant(LoMemVT.getStoreSize(), SL,
1385  PtrVT));
1386 
1387  const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1388  unsigned BaseAlign = Store->getAlignment();
1389  unsigned Size = LoMemVT.getStoreSize();
1390  unsigned HiAlign = MinAlign(BaseAlign, Size);
1391 
1392  SDValue LoStore =
1393  DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
1394  Store->getMemOperand()->getFlags());
1395  SDValue HiStore =
1396  DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size),
1397  HiMemVT, HiAlign, Store->getMemOperand()->getFlags());
1398 
1399  return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
1400 }
1401 
1402 // This is a shortcut for integer division because we have fast i32<->f32
1403 // conversions, and fast f32 reciprocal instructions. The fractional part of a
1404 // float is enough to accurately represent up to a 24-bit signed integer.
1406  bool Sign) const {
1407  SDLoc DL(Op);
1408  EVT VT = Op.getValueType();
1409  SDValue LHS = Op.getOperand(0);
1410  SDValue RHS = Op.getOperand(1);
1411  MVT IntVT = MVT::i32;
1412  MVT FltVT = MVT::f32;
1413 
1414  unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS);
1415  if (LHSSignBits < 9)
1416  return SDValue();
1417 
1418  unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS);
1419  if (RHSSignBits < 9)
1420  return SDValue();
1421 
1422  unsigned BitSize = VT.getSizeInBits();
1423  unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
1424  unsigned DivBits = BitSize - SignBits;
1425  if (Sign)
1426  ++DivBits;
1427 
1430 
1431  SDValue jq = DAG.getConstant(1, DL, IntVT);
1432 
1433  if (Sign) {
1434  // char|short jq = ia ^ ib;
1435  jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
1436 
1437  // jq = jq >> (bitsize - 2)
1438  jq = DAG.getNode(ISD::SRA, DL, VT, jq,
1439  DAG.getConstant(BitSize - 2, DL, VT));
1440 
1441  // jq = jq | 0x1
1442  jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
1443  }
1444 
1445  // int ia = (int)LHS;
1446  SDValue ia = LHS;
1447 
1448  // int ib, (int)RHS;
1449  SDValue ib = RHS;
1450 
1451  // float fa = (float)ia;
1452  SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
1453 
1454  // float fb = (float)ib;
1455  SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
1456 
1457  SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
1458  fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
1459 
1460  // fq = trunc(fq);
1461  fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
1462 
1463  // float fqneg = -fq;
1464  SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
1465 
1466  // float fr = mad(fqneg, fb, fa);
1467  unsigned OpCode = Subtarget->hasFP32Denormals() ?
1469  (unsigned)ISD::FMAD;
1470  SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
1471 
1472  // int iq = (int)fq;
1473  SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
1474 
1475  // fr = fabs(fr);
1476  fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);
1477 
1478  // fb = fabs(fb);
1479  fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);
1480 
1481  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
1482 
1483  // int cv = fr >= fb;
1484  SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
1485 
1486  // jq = (cv ? jq : 0);
1487  jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));
1488 
1489  // dst = iq + jq;
1490  SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
1491 
1492  // Rem needs compensation, it's easier to recompute it
1493  SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
1494  Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
1495 
1496  // Truncate to number of bits this divide really is.
1497  if (Sign) {
1498  SDValue InRegSize
1499  = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits));
1500  Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize);
1501  Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize);
1502  } else {
1503  SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
1504  Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask);
1505  Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask);
1506  }
1507 
1508  return DAG.getMergeValues({ Div, Rem }, DL);
1509 }
1510 
1512  SelectionDAG &DAG,
1514  SDLoc DL(Op);
1515  EVT VT = Op.getValueType();
1516 
1517  assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
1518 
1519  EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
1520 
1521  SDValue One = DAG.getConstant(1, DL, HalfVT);
1522  SDValue Zero = DAG.getConstant(0, DL, HalfVT);
1523 
1524  //HiLo split
1525  SDValue LHS = Op.getOperand(0);
1526  SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1527  SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, One);
1528 
1529  SDValue RHS = Op.getOperand(1);
1530  SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1531  SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, One);
1532 
1533  if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) &&
1534  DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) {
1535 
1536  SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1537  LHS_Lo, RHS_Lo);
1538 
1539  SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
1540  SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});
1541 
1542  Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV));
1543  Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM));
1544  return;
1545  }
1546 
1547  if (isTypeLegal(MVT::i64)) {
1548  // Compute denominator reciprocal.
1549  unsigned FMAD = Subtarget->hasFP32Denormals() ?
1551  (unsigned)ISD::FMAD;
1552 
1553  SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
1554  SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
1555  SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
1556  DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
1557  Cvt_Lo);
1558  SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
1559  SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
1560  DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
1561  SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
1562  DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
1563  SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
1564  SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
1565  DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
1566  Mul1);
1567  SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
1568  SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
1569  SDValue Rcp64 = DAG.getBitcast(VT,
1570  DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));
1571 
1572  SDValue Zero64 = DAG.getConstant(0, DL, VT);
1573  SDValue One64 = DAG.getConstant(1, DL, VT);
1574  SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
1575  SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1);
1576 
1577  SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
1578  SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
1579  SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
1580  SDValue Mulhi1_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1581  Zero);
1582  SDValue Mulhi1_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1583  One);
1584 
1585  SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo,
1586  Mulhi1_Lo, Zero1);
1587  SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi,
1588  Mulhi1_Hi, Add1_Lo.getValue(1));
1589  SDValue Add1_HiNc = DAG.getNode(ISD::ADD, DL, HalfVT, Rcp_Hi, Mulhi1_Hi);
1590  SDValue Add1 = DAG.getBitcast(VT,
1591  DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
1592 
1593  SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
1594  SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
1595  SDValue Mulhi2_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1596  Zero);
1597  SDValue Mulhi2_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1598  One);
1599 
1600  SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo,
1601  Mulhi2_Lo, Zero1);
1602  SDValue Add2_HiC = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_HiNc,
1603  Mulhi2_Hi, Add1_Lo.getValue(1));
1604  SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add2_HiC,
1605  Zero, Add2_Lo.getValue(1));
1606  SDValue Add2 = DAG.getBitcast(VT,
1607  DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
1608  SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
1609 
1610  SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
1611 
1612  SDValue Mul3_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, Zero);
1613  SDValue Mul3_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, One);
1614  SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo,
1615  Mul3_Lo, Zero1);
1616  SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi,
1617  Mul3_Hi, Sub1_Lo.getValue(1));
1618  SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
1619  SDValue Sub1 = DAG.getBitcast(VT,
1620  DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));
1621 
1622  SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
1623  SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
1624  ISD::SETUGE);
1625  SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
1626  ISD::SETUGE);
1627  SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
1628 
1629  // TODO: Here and below portions of the code can be enclosed into if/endif.
1630  // Currently control flow is unconditional and we have 4 selects after
1631  // potential endif to substitute PHIs.
1632 
1633  // if C3 != 0 ...
1634  SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo,
1635  RHS_Lo, Zero1);
1636  SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi,
1637  RHS_Hi, Sub1_Lo.getValue(1));
1638  SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1639  Zero, Sub2_Lo.getValue(1));
1640  SDValue Sub2 = DAG.getBitcast(VT,
1641  DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
1642 
1643  SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
1644 
1645  SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
1646  ISD::SETUGE);
1647  SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
1648  ISD::SETUGE);
1649  SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
1650 
1651  // if (C6 != 0)
1652  SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
1653 
1654  SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo,
1655  RHS_Lo, Zero1);
1656  SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1657  RHS_Hi, Sub2_Lo.getValue(1));
1658  SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi,
1659  Zero, Sub3_Lo.getValue(1));
1660  SDValue Sub3 = DAG.getBitcast(VT,
1661  DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
1662 
1663  // endif C6
1664  // endif C3
1665 
1666  SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
1667  SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
1668 
1669  SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
1670  SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
1671 
1672  Results.push_back(Div);
1673  Results.push_back(Rem);
1674 
1675  return;
1676  }
1677 
1678  // r600 expandion.
1679  // Get Speculative values
1680  SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
1681  SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
1682 
1683  SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
1684  SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
1685  REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);
1686 
1687  SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
1688  SDValue DIV_Lo = Zero;
1689 
1690  const unsigned halfBitWidth = HalfVT.getSizeInBits();
1691 
1692  for (unsigned i = 0; i < halfBitWidth; ++i) {
1693  const unsigned bitPos = halfBitWidth - i - 1;
1694  SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
1695  // Get value of high bit
1696  SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
1697  HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One);
1698  HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
1699 
1700  // Shift
1701  REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
1702  // Add LHS high bit
1703  REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
1704 
1705  SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
1706  SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);
1707 
1708  DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
1709 
1710  // Update REM
1711  SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
1712  REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE);
1713  }
1714 
1715  SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
1716  DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV);
1717  Results.push_back(DIV);
1718  Results.push_back(REM);
1719 }
1720 
1722  SelectionDAG &DAG) const {
1723  SDLoc DL(Op);
1724  EVT VT = Op.getValueType();
1725 
1726  if (VT == MVT::i64) {
1728  LowerUDIVREM64(Op, DAG, Results);
1729  return DAG.getMergeValues(Results, DL);
1730  }
1731 
1732  if (VT == MVT::i32) {
1733  if (SDValue Res = LowerDIVREM24(Op, DAG, false))
1734  return Res;
1735  }
1736 
1737  SDValue Num = Op.getOperand(0);
1738  SDValue Den = Op.getOperand(1);
1739 
1740  // RCP = URECIP(Den) = 2^32 / Den + e
1741  // e is rounding error.
1742  SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
1743 
1744  // RCP_LO = mul(RCP, Den) */
1745  SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den);
1746 
1747  // RCP_HI = mulhu (RCP, Den) */
1748  SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
1749 
1750  // NEG_RCP_LO = -RCP_LO
1751  SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
1752  RCP_LO);
1753 
1754  // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
1755  SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
1756  NEG_RCP_LO, RCP_LO,
1757  ISD::SETEQ);
1758  // Calculate the rounding error from the URECIP instruction
1759  // E = mulhu(ABS_RCP_LO, RCP)
1760  SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
1761 
1762  // RCP_A_E = RCP + E
1763  SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
1764 
1765  // RCP_S_E = RCP - E
1766  SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
1767 
1768  // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
1769  SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
1770  RCP_A_E, RCP_S_E,
1771  ISD::SETEQ);
1772  // Quotient = mulhu(Tmp0, Num)
1773  SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
1774 
1775  // Num_S_Remainder = Quotient * Den
1776  SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den);
1777 
1778  // Remainder = Num - Num_S_Remainder
1779  SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
1780 
1781  // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
1782  SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
1783  DAG.getConstant(-1, DL, VT),
1784  DAG.getConstant(0, DL, VT),
1785  ISD::SETUGE);
1786  // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
1787  SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
1788  Num_S_Remainder,
1789  DAG.getConstant(-1, DL, VT),
1790  DAG.getConstant(0, DL, VT),
1791  ISD::SETUGE);
1792  // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
1793  SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
1794  Remainder_GE_Zero);
1795 
1796  // Calculate Division result:
1797 
1798  // Quotient_A_One = Quotient + 1
1799  SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
1800  DAG.getConstant(1, DL, VT));
1801 
1802  // Quotient_S_One = Quotient - 1
1803  SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
1804  DAG.getConstant(1, DL, VT));
1805 
1806  // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
1807  SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
1808  Quotient, Quotient_A_One, ISD::SETEQ);
1809 
1810  // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
1811  Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
1812  Quotient_S_One, Div, ISD::SETEQ);
1813 
1814  // Calculate Rem result:
1815 
1816  // Remainder_S_Den = Remainder - Den
1817  SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
1818 
1819  // Remainder_A_Den = Remainder + Den
1820  SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
1821 
1822  // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
1823  SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
1824  Remainder, Remainder_S_Den, ISD::SETEQ);
1825 
1826  // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
1827  Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
1828  Remainder_A_Den, Rem, ISD::SETEQ);
1829  SDValue Ops[2] = {
1830  Div,
1831  Rem
1832  };
1833  return DAG.getMergeValues(Ops, DL);
1834 }
1835 
1837  SelectionDAG &DAG) const {
1838  SDLoc DL(Op);
1839  EVT VT = Op.getValueType();
1840 
1841  SDValue LHS = Op.getOperand(0);
1842  SDValue RHS = Op.getOperand(1);
1843 
1844  SDValue Zero = DAG.getConstant(0, DL, VT);
1845  SDValue NegOne = DAG.getConstant(-1, DL, VT);
1846 
1847  if (VT == MVT::i32) {
1848  if (SDValue Res = LowerDIVREM24(Op, DAG, true))
1849  return Res;
1850  }
1851 
1852  if (VT == MVT::i64 &&
1853  DAG.ComputeNumSignBits(LHS) > 32 &&
1854  DAG.ComputeNumSignBits(RHS) > 32) {
1855  EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
1856 
1857  //HiLo split
1858  SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1859  SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1860  SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1861  LHS_Lo, RHS_Lo);
1862  SDValue Res[2] = {
1863  DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
1864  DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
1865  };
1866  return DAG.getMergeValues(Res, DL);
1867  }
1868 
1869  SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
1870  SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
1871  SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
1872  SDValue RSign = LHSign; // Remainder sign is the same as LHS
1873 
1874  LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
1875  RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
1876 
1877  LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
1878  RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
1879 
1880  SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
1881  SDValue Rem = Div.getValue(1);
1882 
1883  Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
1884  Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
1885 
1886  Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
1887  Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
1888 
1889  SDValue Res[2] = {
1890  Div,
1891  Rem
1892  };
1893  return DAG.getMergeValues(Res, DL);
1894 }
1895 
1896 // (frem x, y) -> (fsub x, (fmul (ftrunc (fdiv x, y)), y))
1898  SDLoc SL(Op);
1899  EVT VT = Op.getValueType();
1900  SDValue X = Op.getOperand(0);
1901  SDValue Y = Op.getOperand(1);
1902 
1903  // TODO: Should this propagate fast-math-flags?
1904 
1905  SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
1906  SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
1907  SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
1908 
1909  return DAG.getNode(ISD::FSUB, SL, VT, X, Mul);
1910 }
1911 
1913  SDLoc SL(Op);
1914  SDValue Src = Op.getOperand(0);
1915 
1916  // result = trunc(src)
1917  // if (src > 0.0 && src != result)
1918  // result += 1.0
1919 
1920  SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
1921 
1922  const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
1923  const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
1924 
1925  EVT SetCCVT =
1927 
1928  SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
1929  SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
1930  SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
1931 
1932  SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
1933  // TODO: Should this propagate fast-math-flags?
1934  return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
1935 }
1936 
1938  SelectionDAG &DAG) {
1939  const unsigned FractBits = 52;
1940  const unsigned ExpBits = 11;
1941 
1942  SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
1943  Hi,
1944  DAG.getConstant(FractBits - 32, SL, MVT::i32),
1945  DAG.getConstant(ExpBits, SL, MVT::i32));
1946  SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
1947  DAG.getConstant(1023, SL, MVT::i32));
1948 
1949  return Exp;
1950 }
1951 
1953  SDLoc SL(Op);
1954  SDValue Src = Op.getOperand(0);
1955 
1956  assert(Op.getValueType() == MVT::f64);
1957 
1958  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1959  const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1960 
1961  SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
1962 
1963  // Extract the upper half, since this is where we will find the sign and
1964  // exponent.
1965  SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
1966 
1967  SDValue Exp = extractF64Exponent(Hi, SL, DAG);
1968 
1969  const unsigned FractBits = 52;
1970 
1971  // Extract the sign bit.
1972  const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
1973  SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
1974 
1975  // Extend back to to 64-bits.
1976  SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
1977  SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
1978 
1979  SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
1980  const SDValue FractMask
1981  = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
1982 
1983  SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
1984  SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
1985  SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
1986 
1987  EVT SetCCVT =
1989 
1990  const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
1991 
1992  SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
1993  SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
1994 
1995  SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
1996  SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
1997 
1998  return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
1999 }
2000 
2002  SDLoc SL(Op);
2003  SDValue Src = Op.getOperand(0);
2004 
2005  assert(Op.getValueType() == MVT::f64);
2006 
2007  APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2008  SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
2009  SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
2010 
2011  // TODO: Should this propagate fast-math-flags?
2012 
2013  SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
2014  SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
2015 
2016  SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
2017 
2018  APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2019  SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
2020 
2021  EVT SetCCVT =
2023  SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
2024 
2025  return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
2026 }
2027 
2029  // FNEARBYINT and FRINT are the same, except in their handling of FP
2030  // exceptions. Those aren't really meaningful for us, and OpenCL only has
2031  // rint, so just treat them as equivalent.
2032  return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0));
2033 }
2034 
2035 // XXX - May require not supporting f32 denormals?
2036 
2037 // Don't handle v2f16. The extra instructions to scalarize and repack around the
2038 // compare and vselect end up producing worse code than scalarizing the whole
2039 // operation.
2041  SDLoc SL(Op);
2042  SDValue X = Op.getOperand(0);
2043  EVT VT = Op.getValueType();
2044 
2045  SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
2046 
2047  // TODO: Should this propagate fast-math-flags?
2048 
2049  SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
2050 
2051  SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
2052 
2053  const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2054  const SDValue One = DAG.getConstantFP(1.0, SL, VT);
2055  const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
2056 
2057  SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X);
2058 
2059  EVT SetCCVT =
2060  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2061 
2062  SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
2063 
2064  SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero);
2065 
2066  return DAG.getNode(ISD::FADD, SL, VT, T, Sel);
2067 }
2068 
2070  SDLoc SL(Op);
2071  SDValue X = Op.getOperand(0);
2072 
2073  SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X);
2074 
2075  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2076  const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2077  const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32);
2078  const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32);
2079  EVT SetCCVT =
2081 
2082  SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
2083 
2084  SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, One);
2085 
2086  SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2087 
2088  const SDValue Mask = DAG.getConstant(INT64_C(0x000fffffffffffff), SL,
2089  MVT::i64);
2090 
2091  SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp);
2092  SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64,
2093  DAG.getConstant(INT64_C(0x0008000000000000), SL,
2094  MVT::i64),
2095  Exp);
2096 
2097  SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M);
2098  SDValue Tmp1 = DAG.getSetCC(SL, SetCCVT,
2099  DAG.getConstant(0, SL, MVT::i64), Tmp0,
2100  ISD::SETNE);
2101 
2102  SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1,
2103  D, DAG.getConstant(0, SL, MVT::i64));
2104  SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2);
2105 
2106  K = DAG.getNode(ISD::AND, SL, MVT::i64, K, DAG.getNOT(SL, M, MVT::i64));
2107  K = DAG.getNode(ISD::BITCAST, SL, MVT::f64, K);
2108 
2109  SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2110  SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2111  SDValue ExpEqNegOne = DAG.getSetCC(SL, SetCCVT, NegOne, Exp, ISD::SETEQ);
2112 
2113  SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64,
2114  ExpEqNegOne,
2115  DAG.getConstantFP(1.0, SL, MVT::f64),
2116  DAG.getConstantFP(0.0, SL, MVT::f64));
2117 
2118  SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X);
2119 
2120  K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpLt0, S, K);
2121  K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpGt51, X, K);
2122 
2123  return K;
2124 }
2125 
2127  EVT VT = Op.getValueType();
2128 
2129  if (VT == MVT::f32 || VT == MVT::f16)
2130  return LowerFROUND32_16(Op, DAG);
2131 
2132  if (VT == MVT::f64)
2133  return LowerFROUND64(Op, DAG);
2134 
2135  llvm_unreachable("unhandled type");
2136 }
2137 
2139  SDLoc SL(Op);
2140  SDValue Src = Op.getOperand(0);
2141 
2142  // result = trunc(src);
2143  // if (src < 0.0 && src != result)
2144  // result += -1.0.
2145 
2146  SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2147 
2148  const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2149  const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
2150 
2151  EVT SetCCVT =
2153 
2154  SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
2155  SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2156  SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2157 
2158  SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
2159  // TODO: Should this propagate fast-math-flags?
2160  return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2161 }
2162 
2163 static bool isCtlzOpc(unsigned Opc) {
2164  return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF;
2165 }
2166 
2167 static bool isCttzOpc(unsigned Opc) {
2168  return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_UNDEF;
2169 }
2170 
2172  SDLoc SL(Op);
2173  SDValue Src = Op.getOperand(0);
2174  bool ZeroUndef = Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
2176 
2177  unsigned ISDOpc, NewOpc;
2178  if (isCtlzOpc(Op.getOpcode())) {
2179  ISDOpc = ISD::CTLZ_ZERO_UNDEF;
2180  NewOpc = AMDGPUISD::FFBH_U32;
2181  } else if (isCttzOpc(Op.getOpcode())) {
2182  ISDOpc = ISD::CTTZ_ZERO_UNDEF;
2183  NewOpc = AMDGPUISD::FFBL_B32;
2184  } else
2185  llvm_unreachable("Unexpected OPCode!!!");
2186 
2187 
2188  if (ZeroUndef && Src.getValueType() == MVT::i32)
2189  return DAG.getNode(NewOpc, SL, MVT::i32, Src);
2190 
2191  SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2192 
2193  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2194  const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2195 
2196  SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
2197  SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
2198 
2199  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
2200  *DAG.getContext(), MVT::i32);
2201 
2202  SDValue HiOrLo = isCtlzOpc(Op.getOpcode()) ? Hi : Lo;
2203  SDValue Hi0orLo0 = DAG.getSetCC(SL, SetCCVT, HiOrLo, Zero, ISD::SETEQ);
2204 
2205  SDValue OprLo = DAG.getNode(ISDOpc, SL, MVT::i32, Lo);
2206  SDValue OprHi = DAG.getNode(ISDOpc, SL, MVT::i32, Hi);
2207 
2208  const SDValue Bits32 = DAG.getConstant(32, SL, MVT::i32);
2209  SDValue Add, NewOpr;
2210  if (isCtlzOpc(Op.getOpcode())) {
2211  Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprLo, Bits32);
2212  // ctlz(x) = hi_32(x) == 0 ? ctlz(lo_32(x)) + 32 : ctlz(hi_32(x))
2213  NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprHi);
2214  } else {
2215  Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprHi, Bits32);
2216  // cttz(x) = lo_32(x) == 0 ? cttz(hi_32(x)) + 32 : cttz(lo_32(x))
2217  NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprLo);
2218  }
2219 
2220  if (!ZeroUndef) {
2221  // Test if the full 64-bit input is zero.
2222 
2223  // FIXME: DAG combines turn what should be an s_and_b64 into a v_or_b32,
2224  // which we probably don't want.
2225  SDValue LoOrHi = isCtlzOpc(Op.getOpcode()) ? Lo : Hi;
2226  SDValue Lo0OrHi0 = DAG.getSetCC(SL, SetCCVT, LoOrHi, Zero, ISD::SETEQ);
2227  SDValue SrcIsZero = DAG.getNode(ISD::AND, SL, SetCCVT, Lo0OrHi0, Hi0orLo0);
2228 
2229  // TODO: If i64 setcc is half rate, it can result in 1 fewer instruction
2230  // with the same cycles, otherwise it is slower.
2231  // SDValue SrcIsZero = DAG.getSetCC(SL, SetCCVT, Src,
2232  // DAG.getConstant(0, SL, MVT::i64), ISD::SETEQ);
2233 
2234  const SDValue Bits32 = DAG.getConstant(64, SL, MVT::i32);
2235 
2236  // The instruction returns -1 for 0 input, but the defined intrinsic
2237  // behavior is to return the number of bits.
2238  NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32,
2239  SrcIsZero, Bits32, NewOpr);
2240  }
2241 
2242  return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr);
2243 }
2244 
2246  bool Signed) const {
2247  // Unsigned
2248  // cul2f(ulong u)
2249  //{
2250  // uint lz = clz(u);
2251  // uint e = (u != 0) ? 127U + 63U - lz : 0;
2252  // u = (u << lz) & 0x7fffffffffffffffUL;
2253  // ulong t = u & 0xffffffffffUL;
2254  // uint v = (e << 23) | (uint)(u >> 40);
2255  // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
2256  // return as_float(v + r);
2257  //}
2258  // Signed
2259  // cl2f(long l)
2260  //{
2261  // long s = l >> 63;
2262  // float r = cul2f((l + s) ^ s);
2263  // return s ? -r : r;
2264  //}
2265 
2266  SDLoc SL(Op);
2267  SDValue Src = Op.getOperand(0);
2268  SDValue L = Src;
2269 
2270  SDValue S;
2271  if (Signed) {
2272  const SDValue SignBit = DAG.getConstant(63, SL, MVT::i64);
2273  S = DAG.getNode(ISD::SRA, SL, MVT::i64, L, SignBit);
2274 
2275  SDValue LPlusS = DAG.getNode(ISD::ADD, SL, MVT::i64, L, S);
2276  L = DAG.getNode(ISD::XOR, SL, MVT::i64, LPlusS, S);
2277  }
2278 
2279  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
2280  *DAG.getContext(), MVT::f32);
2281 
2282 
2283  SDValue ZeroI32 = DAG.getConstant(0, SL, MVT::i32);
2284  SDValue ZeroI64 = DAG.getConstant(0, SL, MVT::i64);
2285  SDValue LZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i64, L);
2286  LZ = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LZ);
2287 
2288  SDValue K = DAG.getConstant(127U + 63U, SL, MVT::i32);
2289  SDValue E = DAG.getSelect(SL, MVT::i32,
2290  DAG.getSetCC(SL, SetCCVT, L, ZeroI64, ISD::SETNE),
2291  DAG.getNode(ISD::SUB, SL, MVT::i32, K, LZ),
2292  ZeroI32);
2293 
2294  SDValue U = DAG.getNode(ISD::AND, SL, MVT::i64,
2295  DAG.getNode(ISD::SHL, SL, MVT::i64, L, LZ),
2296  DAG.getConstant((-1ULL) >> 1, SL, MVT::i64));
2297 
2298  SDValue T = DAG.getNode(ISD::AND, SL, MVT::i64, U,
2299  DAG.getConstant(0xffffffffffULL, SL, MVT::i64));
2300 
2301  SDValue UShl = DAG.getNode(ISD::SRL, SL, MVT::i64,
2302  U, DAG.getConstant(40, SL, MVT::i64));
2303 
2304  SDValue V = DAG.getNode(ISD::OR, SL, MVT::i32,
2305  DAG.getNode(ISD::SHL, SL, MVT::i32, E, DAG.getConstant(23, SL, MVT::i32)),
2306  DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, UShl));
2307 
2308  SDValue C = DAG.getConstant(0x8000000000ULL, SL, MVT::i64);
2309  SDValue RCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETUGT);
2310  SDValue TCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETEQ);
2311 
2312  SDValue One = DAG.getConstant(1, SL, MVT::i32);
2313 
2314  SDValue VTrunc1 = DAG.getNode(ISD::AND, SL, MVT::i32, V, One);
2315 
2316  SDValue R = DAG.getSelect(SL, MVT::i32,
2317  RCmp,
2318  One,
2319  DAG.getSelect(SL, MVT::i32, TCmp, VTrunc1, ZeroI32));
2320  R = DAG.getNode(ISD::ADD, SL, MVT::i32, V, R);
2321  R = DAG.getNode(ISD::BITCAST, SL, MVT::f32, R);
2322 
2323  if (!Signed)
2324  return R;
2325 
2326  SDValue RNeg = DAG.getNode(ISD::FNEG, SL, MVT::f32, R);
2327  return DAG.getSelect(SL, MVT::f32, DAG.getSExtOrTrunc(S, SL, SetCCVT), RNeg, R);
2328 }
2329 
2331  bool Signed) const {
2332  SDLoc SL(Op);
2333  SDValue Src = Op.getOperand(0);
2334 
2335  SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2336 
2338  DAG.getConstant(0, SL, MVT::i32));
2340  DAG.getConstant(1, SL, MVT::i32));
2341 
2342  SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
2343  SL, MVT::f64, Hi);
2344 
2345  SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
2346 
2347  SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
2348  DAG.getConstant(32, SL, MVT::i32));
2349  // TODO: Should this propagate fast-math-flags?
2350  return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
2351 }
2352 
2354  SelectionDAG &DAG) const {
2355  assert(Op.getOperand(0).getValueType() == MVT::i64 &&
2356  "operation should be legal");
2357 
2358  // TODO: Factor out code common with LowerSINT_TO_FP.
2359 
2360  EVT DestVT = Op.getValueType();
2361  if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
2362  SDLoc DL(Op);
2363  SDValue Src = Op.getOperand(0);
2364 
2365  SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2366  SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2367  SDValue FPRound =
2368  DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
2369 
2370  return FPRound;
2371  }
2372 
2373  if (DestVT == MVT::f32)
2374  return LowerINT_TO_FP32(Op, DAG, false);
2375 
2376  assert(DestVT == MVT::f64);
2377  return LowerINT_TO_FP64(Op, DAG, false);
2378 }
2379 
2381  SelectionDAG &DAG) const {
2382  assert(Op.getOperand(0).getValueType() == MVT::i64 &&
2383  "operation should be legal");
2384 
2385  // TODO: Factor out code common with LowerUINT_TO_FP.
2386 
2387  EVT DestVT = Op.getValueType();
2388  if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
2389  SDLoc DL(Op);
2390  SDValue Src = Op.getOperand(0);
2391 
2392  SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2393  SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2394  SDValue FPRound =
2395  DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
2396 
2397  return FPRound;
2398  }
2399 
2400  if (DestVT == MVT::f32)
2401  return LowerINT_TO_FP32(Op, DAG, true);
2402 
2403  assert(DestVT == MVT::f64);
2404  return LowerINT_TO_FP64(Op, DAG, true);
2405 }
2406 
2408  bool Signed) const {
2409  SDLoc SL(Op);
2410 
2411  SDValue Src = Op.getOperand(0);
2412 
2413  SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2414 
2415  SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL,
2416  MVT::f64);
2417  SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL,
2418  MVT::f64);
2419  // TODO: Should this propagate fast-math-flags?
2420  SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0);
2421 
2422  SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul);
2423 
2424 
2425  SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc);
2426 
2427  SDValue Hi = DAG.getNode(Signed ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, SL,
2428  MVT::i32, FloorMul);
2429  SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
2430 
2431  SDValue Result = DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi});
2432 
2433  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result);
2434 }
2435 
2437  SDLoc DL(Op);
2438  SDValue N0 = Op.getOperand(0);
2439 
2440  // Convert to target node to get known bits
2441  if (N0.getValueType() == MVT::f32)
2442  return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0);
2443 
2445  // There is a generic expand for FP_TO_FP16 with unsafe fast math.
2446  return SDValue();
2447  }
2448 
2450 
2451  // f64 -> f16 conversion using round-to-nearest-even rounding mode.
2452  const unsigned ExpMask = 0x7ff;
2453  const unsigned ExpBiasf64 = 1023;
2454  const unsigned ExpBiasf16 = 15;
2455  SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
2456  SDValue One = DAG.getConstant(1, DL, MVT::i32);
2457  SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
2458  SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
2459  DAG.getConstant(32, DL, MVT::i64));
2460  UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
2461  U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
2462  SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2463  DAG.getConstant(20, DL, MVT::i64));
2464  E = DAG.getNode(ISD::AND, DL, MVT::i32, E,
2465  DAG.getConstant(ExpMask, DL, MVT::i32));
2466  // Subtract the fp64 exponent bias (1023) to get the real exponent and
2467  // add the f16 bias (15) to get the biased exponent for the f16 format.
2468  E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
2469  DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));
2470 
2471  SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2472  DAG.getConstant(8, DL, MVT::i32));
2473  M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
2474  DAG.getConstant(0xffe, DL, MVT::i32));
2475 
2476  SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
2477  DAG.getConstant(0x1ff, DL, MVT::i32));
2478  MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U);
2479 
2480  SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
2481  M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set);
2482 
2483  // (M != 0 ? 0x0200 : 0) | 0x7c00;
2484  SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
2485  DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
2486  Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));
2487 
2488  // N = M | (E << 12);
2489  SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2490  DAG.getNode(ISD::SHL, DL, MVT::i32, E,
2491  DAG.getConstant(12, DL, MVT::i32)));
2492 
2493  // B = clamp(1-E, 0, 13);
2494  SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
2495  One, E);
2496  SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
2497  B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
2498  DAG.getConstant(13, DL, MVT::i32));
2499 
2500  SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2501  DAG.getConstant(0x1000, DL, MVT::i32));
2502 
2503  SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
2504  SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
2505  SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
2506  D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1);
2507 
2508  SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
2509  SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
2510  DAG.getConstant(0x7, DL, MVT::i32));
2511  V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
2512  DAG.getConstant(2, DL, MVT::i32));
2513  SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
2514  One, Zero, ISD::SETEQ);
2515  SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
2516  One, Zero, ISD::SETGT);
2517  V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
2518  V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);
2519 
2520  V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
2521  DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
2522  V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
2523  I, V, ISD::SETEQ);
2524 
2525  // Extract the sign bit.
2526  SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2527  DAG.getConstant(16, DL, MVT::i32));
2528  Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
2529  DAG.getConstant(0x8000, DL, MVT::i32));
2530 
2531  V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
2532  return DAG.getZExtOrTrunc(V, DL, Op.getValueType());
2533 }
2534 
2536  SelectionDAG &DAG) const {
2537  SDValue Src = Op.getOperand(0);
2538 
2539  // TODO: Factor out code common with LowerFP_TO_UINT.
2540 
2541  EVT SrcVT = Src.getValueType();
2542  if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
2543  SDLoc DL(Op);
2544 
2545  SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2546  SDValue FpToInt32 =
2547  DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
2548 
2549  return FpToInt32;
2550  }
2551 
2552  if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
2553  return LowerFP64_TO_INT(Op, DAG, true);
2554 
2555  return SDValue();
2556 }
2557 
2559  SelectionDAG &DAG) const {
2560  SDValue Src = Op.getOperand(0);
2561 
2562  // TODO: Factor out code common with LowerFP_TO_SINT.
2563 
2564  EVT SrcVT = Src.getValueType();
2565  if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
2566  SDLoc DL(Op);
2567 
2568  SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2569  SDValue FpToInt32 =
2570  DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
2571 
2572  return FpToInt32;
2573  }
2574 
2575  if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
2576  return LowerFP64_TO_INT(Op, DAG, false);
2577 
2578  return SDValue();
2579 }
2580 
2582  SelectionDAG &DAG) const {
2583  EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2584  MVT VT = Op.getSimpleValueType();
2585  MVT ScalarVT = VT.getScalarType();
2586 
2587  assert(VT.isVector());
2588 
2589  SDValue Src = Op.getOperand(0);
2590  SDLoc DL(Op);
2591 
2592  // TODO: Don't scalarize on Evergreen?
2593  unsigned NElts = VT.getVectorNumElements();
2595  DAG.ExtractVectorElements(Src, Args, 0, NElts);
2596 
2597  SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
2598  for (unsigned I = 0; I < NElts; ++I)
2599  Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
2600 
2601  return DAG.getBuildVector(VT, DL, Args);
2602 }
2603 
2604 //===----------------------------------------------------------------------===//
2605 // Custom DAG optimizations
2606 //===----------------------------------------------------------------------===//
2607 
2608 static bool isU24(SDValue Op, SelectionDAG &DAG) {
2609  return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
2610 }
2611 
2612 static bool isI24(SDValue Op, SelectionDAG &DAG) {
2613  EVT VT = Op.getValueType();
2614  return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
2615  // as unsigned 24-bit values.
2617 }
2618 
2619 static bool simplifyI24(SDNode *Node24, unsigned OpIdx,
2621 
2622  SelectionDAG &DAG = DCI.DAG;
2623  SDValue Op = Node24->getOperand(OpIdx);
2624  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2625  EVT VT = Op.getValueType();
2626 
2627  APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24);
2628  APInt KnownZero, KnownOne;
2629  TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
2630  if (TLI.SimplifyDemandedBits(Node24, OpIdx, Demanded, DCI, TLO))
2631  return true;
2632 
2633  return false;
2634 }
2635 
2636 template <typename IntTy>
2638  uint32_t Width, const SDLoc &DL) {
2639  if (Width + Offset < 32) {
2640  uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
2641  IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
2642  return DAG.getConstant(Result, DL, MVT::i32);
2643  }
2644 
2645  return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
2646 }
2647 
2648 static bool hasVolatileUser(SDNode *Val) {
2649  for (SDNode *U : Val->uses()) {
2650  if (MemSDNode *M = dyn_cast<MemSDNode>(U)) {
2651  if (M->isVolatile())
2652  return true;
2653  }
2654  }
2655 
2656  return false;
2657 }
2658 
2660  // i32 vectors are the canonical memory type.
2661  if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
2662  return false;
2663 
2664  if (!VT.isByteSized())
2665  return false;
2666 
2667  unsigned Size = VT.getStoreSize();
2668 
2669  if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
2670  return false;
2671 
2672  if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
2673  return false;
2674 
2675  return true;
2676 }
2677 
2678 // Replace load of an illegal type with a store of a bitcast to a friendlier
2679 // type.
2681  DAGCombinerInfo &DCI) const {
2682  if (!DCI.isBeforeLegalize())
2683  return SDValue();
2684 
2685  LoadSDNode *LN = cast<LoadSDNode>(N);
2686  if (LN->isVolatile() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN))
2687  return SDValue();
2688 
2689  SDLoc SL(N);
2690  SelectionDAG &DAG = DCI.DAG;
2691  EVT VT = LN->getMemoryVT();
2692 
2693  unsigned Size = VT.getStoreSize();
2694  unsigned Align = LN->getAlignment();
2695  if (Align < Size && isTypeLegal(VT)) {
2696  bool IsFast;
2697  unsigned AS = LN->getAddressSpace();
2698 
2699  // Expand unaligned loads earlier than legalization. Due to visitation order
2700  // problems during legalization, the emitted instructions to pack and unpack
2701  // the bytes again are not eliminated in the case of an unaligned copy.
2702  if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) {
2703  if (VT.isVector())
2704  return scalarizeVectorLoad(LN, DAG);
2705 
2706  SDValue Ops[2];
2707  std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG);
2708  return DAG.getMergeValues(Ops, SDLoc(N));
2709  }
2710 
2711  if (!IsFast)
2712  return SDValue();
2713  }
2714 
2715  if (!shouldCombineMemoryType(VT))
2716  return SDValue();
2717 
2718  EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
2719 
2720  SDValue NewLoad
2721  = DAG.getLoad(NewVT, SL, LN->getChain(),
2722  LN->getBasePtr(), LN->getMemOperand());
2723 
2724  SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
2725  DCI.CombineTo(N, BC, NewLoad.getValue(1));
2726  return SDValue(N, 0);
2727 }
2728 
2729 // Replace store of an illegal type with a store of a bitcast to a friendlier
2730 // type.
2732  DAGCombinerInfo &DCI) const {
2733  if (!DCI.isBeforeLegalize())
2734  return SDValue();
2735 
2736  StoreSDNode *SN = cast<StoreSDNode>(N);
2737  if (SN->isVolatile() || !ISD::isNormalStore(SN))
2738  return SDValue();
2739 
2740  EVT VT = SN->getMemoryVT();
2741  unsigned Size = VT.getStoreSize();
2742 
2743  SDLoc SL(N);
2744  SelectionDAG &DAG = DCI.DAG;
2745  unsigned Align = SN->getAlignment();
2746  if (Align < Size && isTypeLegal(VT)) {
2747  bool IsFast;
2748  unsigned AS = SN->getAddressSpace();
2749 
2750  // Expand unaligned stores earlier than legalization. Due to visitation
2751  // order problems during legalization, the emitted instructions to pack and
2752  // unpack the bytes again are not eliminated in the case of an unaligned
2753  // copy.
2754  if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) {
2755  if (VT.isVector())
2756  return scalarizeVectorStore(SN, DAG);
2757 
2758  return expandUnalignedStore(SN, DAG);
2759  }
2760 
2761  if (!IsFast)
2762  return SDValue();
2763  }
2764 
2765  if (!shouldCombineMemoryType(VT))
2766  return SDValue();
2767 
2768  EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
2769  SDValue Val = SN->getValue();
2770 
2771  //DCI.AddToWorklist(Val.getNode());
2772 
2773  bool OtherUses = !Val.hasOneUse();
2774  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
2775  if (OtherUses) {
2776  SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
2777  DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
2778  }
2779 
2780  return DAG.getStore(SN->getChain(), SL, CastVal,
2781  SN->getBasePtr(), SN->getMemOperand());
2782 }
2783 
2785  DAGCombinerInfo &DCI) const {
2787  if (!CSrc)
2788  return SDValue();
2789 
2790  const APFloat &F = CSrc->getValueAPF();
2792  APFloat::cmpResult Cmp0 = F.compare(Zero);
2793  if (Cmp0 == APFloat::cmpLessThan ||
2795  return DCI.DAG.getConstantFP(Zero, SDLoc(N), N->getValueType(0));
2796  }
2797 
2798  APFloat One(F.getSemantics(), "1.0");
2799  APFloat::cmpResult Cmp1 = F.compare(One);
2800  if (Cmp1 == APFloat::cmpGreaterThan)
2801  return DCI.DAG.getConstantFP(One, SDLoc(N), N->getValueType(0));
2802 
2803  return SDValue(CSrc, 0);
2804 }
2805 
2806 // FIXME: This should go in generic DAG combiner with an isTruncateFree check,
2807 // but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
2808 // issues.
2810  DAGCombinerInfo &DCI) const {
2811  SelectionDAG &DAG = DCI.DAG;
2812  SDValue N0 = N->getOperand(0);
2813 
2814  // (vt2 (assertzext (truncate vt0:x), vt1)) ->
2815  // (vt2 (truncate (assertzext vt0:x, vt1)))
2816  if (N0.getOpcode() == ISD::TRUNCATE) {
2817  SDValue N1 = N->getOperand(1);
2818  EVT ExtVT = cast<VTSDNode>(N1)->getVT();
2819  SDLoc SL(N);
2820 
2821  SDValue Src = N0.getOperand(0);
2822  EVT SrcVT = Src.getValueType();
2823  if (SrcVT.bitsGE(ExtVT)) {
2824  SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
2825  return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg);
2826  }
2827  }
2828 
2829  return SDValue();
2830 }
2831 /// Split the 64-bit value \p LHS into two 32-bit components, and perform the
2832 /// binary operation \p Opc to it with the corresponding constant operands.
2834  DAGCombinerInfo &DCI, const SDLoc &SL,
2835  unsigned Opc, SDValue LHS,
2836  uint32_t ValLo, uint32_t ValHi) const {
2837  SelectionDAG &DAG = DCI.DAG;
2838  SDValue Lo, Hi;
2839  std::tie(Lo, Hi) = split64BitValue(LHS, DAG);
2840 
2841  SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
2842  SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
2843 
2844  SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
2845  SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
2846 
2847  // Re-visit the ands. It's possible we eliminated one of them and it could
2848  // simplify the vector.
2849  DCI.AddToWorklist(Lo.getNode());
2850  DCI.AddToWorklist(Hi.getNode());
2851 
2852  SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
2853  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
2854 }
2855 
2857  DAGCombinerInfo &DCI) const {
2858  EVT VT = N->getValueType(0);
2859 
2861  if (!RHS)
2862  return SDValue();
2863 
2864  SDValue LHS = N->getOperand(0);
2865  unsigned RHSVal = RHS->getZExtValue();
2866  if (!RHSVal)
2867  return LHS;
2868 
2869  SDLoc SL(N);
2870  SelectionDAG &DAG = DCI.DAG;
2871 
2872  switch (LHS->getOpcode()) {
2873  default:
2874  break;
2875  case ISD::ZERO_EXTEND:
2876  case ISD::SIGN_EXTEND:
2877  case ISD::ANY_EXTEND: {
2878  SDValue X = LHS->getOperand(0);
2879 
2880  if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
2882  // Prefer build_vector as the canonical form if packed types are legal.
2883  // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
2884  SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
2885  { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) });
2886  return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
2887  }
2888 
2889  // shl (ext x) => zext (shl x), if shift does not overflow int
2890  if (VT != MVT::i64)
2891  break;
2892  KnownBits Known;
2893  DAG.computeKnownBits(X, Known);
2894  unsigned LZ = Known.countMinLeadingZeros();
2895  if (LZ < RHSVal)
2896  break;
2897  EVT XVT = X.getValueType();
2898  SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
2899  return DAG.getZExtOrTrunc(Shl, SL, VT);
2900  }
2901  }
2902 
2903  if (VT != MVT::i64)
2904  return SDValue();
2905 
2906  // i64 (shl x, C) -> (build_pair 0, (shl x, C -32))
2907 
2908  // On some subtargets, 64-bit shift is a quarter rate instruction. In the
2909  // common case, splitting this into a move and a 32-bit shift is faster and
2910  // the same code size.
2911  if (RHSVal < 32)
2912  return SDValue();
2913 
2914  SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
2915 
2916  SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
2917  SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
2918 
2919  const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2920 
2921  SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
2922  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
2923 }
2924 
2926  DAGCombinerInfo &DCI) const {
2927  if (N->getValueType(0) != MVT::i64)
2928  return SDValue();
2929 
2930  const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
2931  if (!RHS)
2932  return SDValue();
2933 
2934  SelectionDAG &DAG = DCI.DAG;
2935  SDLoc SL(N);
2936  unsigned RHSVal = RHS->getZExtValue();
2937 
2938  // (sra i64:x, 32) -> build_pair x, (sra hi_32(x), 31)
2939  if (RHSVal == 32) {
2940  SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
2941  SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
2942  DAG.getConstant(31, SL, MVT::i32));
2943 
2944  SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
2945  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
2946  }
2947 
2948  // (sra i64:x, 63) -> build_pair (sra hi_32(x), 31), (sra hi_32(x), 31)
2949  if (RHSVal == 63) {
2950  SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
2951  SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
2952  DAG.getConstant(31, SL, MVT::i32));
2953  SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
2954  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
2955  }
2956 
2957  return SDValue();
2958 }
2959 
2961  DAGCombinerInfo &DCI) const {
2962  if (N->getValueType(0) != MVT::i64)
2963  return SDValue();
2964 
2965  const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
2966  if (!RHS)
2967  return SDValue();
2968 
2969  unsigned ShiftAmt = RHS->getZExtValue();
2970  if (ShiftAmt < 32)
2971  return SDValue();
2972 
2973  // srl i64:x, C for C >= 32
2974  // =>
2975  // build_pair (srl hi_32(x), C - 32), 0
2976 
2977  SelectionDAG &DAG = DCI.DAG;
2978  SDLoc SL(N);
2979 
2980  SDValue One = DAG.getConstant(1, SL, MVT::i32);
2981  SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2982 
2983  SDValue VecOp = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, N->getOperand(0));
2985  VecOp, One);
2986 
2987  SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
2988  SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
2989 
2990  SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
2991 
2992  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
2993 }
2994 
2995 // We need to specifically handle i64 mul here to avoid unnecessary conversion
2996 // instructions. If we only match on the legalized i64 mul expansion,
2997 // SimplifyDemandedBits will be unable to remove them because there will be
2998 // multiple uses due to the separate mul + mulh[su].
2999 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
3000  SDValue N0, SDValue N1, unsigned Size, bool Signed) {
3001  if (Size <= 32) {
3002  unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
3003  return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
3004  }
3005 
3006  // Because we want to eliminate extension instructions before the
3007  // operation, we need to create a single user here (i.e. not the separate
3008  // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it.
3009 
3010  unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24;
3011 
3012  SDValue Mul = DAG.getNode(MulOpc, SL,
3013  DAG.getVTList(MVT::i32, MVT::i32), N0, N1);
3014 
3015  return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64,
3016  Mul.getValue(0), Mul.getValue(1));
3017 }
3018 
3020  DAGCombinerInfo &DCI) const {
3021  EVT VT = N->getValueType(0);
3022 
3023  unsigned Size = VT.getSizeInBits();
3024  if (VT.isVector() || Size > 64)
3025  return SDValue();
3026 
3027  // There are i16 integer mul/mad.
3029  return SDValue();
3030 
3031  SelectionDAG &DAG = DCI.DAG;
3032  SDLoc DL(N);
3033 
3034  SDValue N0 = N->getOperand(0);
3035  SDValue N1 = N->getOperand(1);
3036  SDValue Mul;
3037 
3038  if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
3039  N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
3040  N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
3041  Mul = getMul24(DAG, DL, N0, N1, Size, false);
3042  } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
3043  N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
3044  N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
3045  Mul = getMul24(DAG, DL, N0, N1, Size, true);
3046  } else {
3047  return SDValue();
3048  }
3049 
3050  // We need to use sext even for MUL_U24, because MUL_U24 is used
3051  // for signed multiply of 8 and 16-bit types.
3052  return DAG.getSExtOrTrunc(Mul, DL, VT);
3053 }
3054 
3056  DAGCombinerInfo &DCI) const {
3057  EVT VT = N->getValueType(0);
3058 
3059  if (!Subtarget->hasMulI24() || VT.isVector())
3060  return SDValue();
3061 
3062  SelectionDAG &DAG = DCI.DAG;
3063  SDLoc DL(N);
3064 
3065  SDValue N0 = N->getOperand(0);
3066  SDValue N1 = N->getOperand(1);
3067 
3068  if (!isI24(N0, DAG) || !isI24(N1, DAG))
3069  return SDValue();
3070 
3071  N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
3072  N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
3073 
3074  SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
3075  DCI.AddToWorklist(Mulhi.getNode());
3076  return DAG.getSExtOrTrunc(Mulhi, DL, VT);
3077 }
3078 
3080  DAGCombinerInfo &DCI) const {
3081  EVT VT = N->getValueType(0);
3082 
3083  if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
3084  return SDValue();
3085 
3086  SelectionDAG &DAG = DCI.DAG;
3087  SDLoc DL(N);
3088 
3089  SDValue N0 = N->getOperand(0);
3090  SDValue N1 = N->getOperand(1);
3091 
3092  if (!isU24(N0, DAG) || !isU24(N1, DAG))
3093  return SDValue();
3094 
3095  N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
3096  N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
3097 
3098  SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
3099  DCI.AddToWorklist(Mulhi.getNode());
3100  return DAG.getZExtOrTrunc(Mulhi, DL, VT);
3101 }
3102 
3104  SDNode *N, DAGCombinerInfo &DCI) const {
3105  SelectionDAG &DAG = DCI.DAG;
3106 
3107  // Simplify demanded bits before splitting into multiple users.
3108  if (simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI))
3109  return SDValue();
3110 
3111  SDValue N0 = N->getOperand(0);
3112  SDValue N1 = N->getOperand(1);
3113 
3114  bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24);
3115 
3116  unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
3117  unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
3118 
3119  SDLoc SL(N);
3120 
3121  SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
3122  SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
3123  return DAG.getMergeValues({ MulLo, MulHi }, SL);
3124 }
3125 
3126 static bool isNegativeOne(SDValue Val) {
3127  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val))
3128  return C->isAllOnesValue();
3129  return false;
3130 }
3131 
3132 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
3133  SDValue Op,
3134  const SDLoc &DL,
3135  unsigned Opc) const {
3136  EVT VT = Op.getValueType();
3137  EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT);
3138  if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() &&
3139  LegalVT != MVT::i16))
3140  return SDValue();
3141 
3142  if (VT != MVT::i32)
3143  Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op);
3144 
3145  SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
3146  if (VT != MVT::i32)
3147  FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX);
3148 
3149  return FFBX;
3150 }
3151 
3152 // The native instructions return -1 on 0 input. Optimize out a select that
3153 // produces -1 on 0.
3154 //
3155 // TODO: If zero is not undef, we could also do this if the output is compared
3156 // against the bitwidth.
3157 //
3158 // TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
3160  SDValue LHS, SDValue RHS,
3161  DAGCombinerInfo &DCI) const {
3162  ConstantSDNode *CmpRhs = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
3163  if (!CmpRhs || !CmpRhs->isNullValue())
3164  return SDValue();
3165 
3166  SelectionDAG &DAG = DCI.DAG;
3167  ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
3168  SDValue CmpLHS = Cond.getOperand(0);
3169 
3170  unsigned Opc = isCttzOpc(RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 :
3172 
3173  // select (setcc x, 0, eq), -1, (ctlz_zero_undef x) -> ffbh_u32 x
3174  // select (setcc x, 0, eq), -1, (cttz_zero_undef x) -> ffbl_u32 x
3175  if (CCOpcode == ISD::SETEQ &&
3176  (isCtlzOpc(RHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
3177  RHS.getOperand(0) == CmpLHS &&
3178  isNegativeOne(LHS)) {
3179  return getFFBX_U32(DAG, CmpLHS, SL, Opc);
3180  }
3181 
3182  // select (setcc x, 0, ne), (ctlz_zero_undef x), -1 -> ffbh_u32 x
3183  // select (setcc x, 0, ne), (cttz_zero_undef x), -1 -> ffbl_u32 x
3184  if (CCOpcode == ISD::SETNE &&
3185  (isCtlzOpc(LHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
3186  LHS.getOperand(0) == CmpLHS &&
3187  isNegativeOne(RHS)) {
3188  return getFFBX_U32(DAG, CmpLHS, SL, Opc);
3189  }
3190 
3191  return SDValue();
3192 }
3193 
3195  unsigned Op,
3196  const SDLoc &SL,
3197  SDValue Cond,
3198  SDValue N1,
3199  SDValue N2) {
3200  SelectionDAG &DAG = DCI.DAG;
3201  EVT VT = N1.getValueType();
3202 
3203  SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
3204  N1.getOperand(0), N2.getOperand(0));
3205  DCI.AddToWorklist(NewSelect.getNode());
3206  return DAG.getNode(Op, SL, VT, NewSelect);
3207 }
3208 
3209 // Pull a free FP operation out of a select so it may fold into uses.
3210 //
3211 // select c, (fneg x), (fneg y) -> fneg (select c, x, y)
3212 // select c, (fneg x), k -> fneg (select c, x, (fneg k))
3213 //
3214 // select c, (fabs x), (fabs y) -> fabs (select c, x, y)
3215 // select c, (fabs x), +k -> fabs (select c, x, k)
3217  SDValue N) {
3218  SelectionDAG &DAG = DCI.DAG;
3219  SDValue Cond = N.getOperand(0);
3220  SDValue LHS = N.getOperand(1);
3221  SDValue RHS = N.getOperand(2);
3222 
3223  EVT VT = N.getValueType();
3224  if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
3225  (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
3226  return distributeOpThroughSelect(DCI, LHS.getOpcode(),
3227  SDLoc(N), Cond, LHS, RHS);
3228  }
3229 
3230  bool Inv = false;
3231  if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
3232  std::swap(LHS, RHS);
3233  Inv = true;
3234  }
3235 
3236  // TODO: Support vector constants.
3238  if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS) {
3239  SDLoc SL(N);
3240  // If one side is an fneg/fabs and the other is a constant, we can push the
3241  // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
3242  SDValue NewLHS = LHS.getOperand(0);
3243  SDValue NewRHS = RHS;
3244 
3245  // Careful: if the neg can be folded up, don't try to pull it back down.
3246  bool ShouldFoldNeg = true;
3247 
3248  if (NewLHS.hasOneUse()) {
3249  unsigned Opc = NewLHS.getOpcode();
3250  if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(Opc))
3251  ShouldFoldNeg = false;
3252  if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
3253  ShouldFoldNeg = false;
3254  }
3255 
3256  if (ShouldFoldNeg) {
3257  if (LHS.getOpcode() == ISD::FNEG)
3258  NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3259  else if (CRHS->isNegative())
3260  return SDValue();
3261 
3262  if (Inv)
3263  std::swap(NewLHS, NewRHS);
3264 
3265  SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
3266  Cond, NewLHS, NewRHS);
3267  DCI.AddToWorklist(NewSelect.getNode());
3268  return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
3269  }
3270  }
3271 
3272  return SDValue();
3273 }
3274 
3275 
3277  DAGCombinerInfo &DCI) const {
3278  if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0)))
3279  return Folded;
3280 
3281  SDValue Cond = N->getOperand(0);
3282  if (Cond.getOpcode() != ISD::SETCC)
3283  return SDValue();
3284 
3285  EVT VT = N->getValueType(0);
3286  SDValue LHS = Cond.getOperand(0);
3287  SDValue RHS = Cond.getOperand(1);
3288  SDValue CC = Cond.getOperand(2);
3289 
3290  SDValue True = N->getOperand(1);
3291  SDValue False = N->getOperand(2);
3292 
3293  if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
3294  SelectionDAG &DAG = DCI.DAG;
3295  if ((DAG.isConstantValueOfAnyType(True) ||
3296  DAG.isConstantValueOfAnyType(True)) &&
3297  (!DAG.isConstantValueOfAnyType(False) &&
3298  !DAG.isConstantValueOfAnyType(False))) {
3299  // Swap cmp + select pair to move constant to false input.
3300  // This will allow using VOPC cndmasks more often.
3301  // select (setcc x, y), k, x -> select (setcc y, x) x, x
3302 
3303  SDLoc SL(N);
3304  ISD::CondCode NewCC = getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
3305  LHS.getValueType().isInteger());
3306 
3307  SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
3308  return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
3309  }
3310 
3311  if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
3312  SDValue MinMax
3313  = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
3314  // Revisit this node so we can catch min3/max3/med3 patterns.
3315  //DCI.AddToWorklist(MinMax.getNode());
3316  return MinMax;
3317  }
3318  }
3319 
3320  // There's no reason to not do this if the condition has other uses.
3321  return performCtlz_CttzCombine(SDLoc(N), Cond, True, False, DCI);
3322 }
3323 
3325  if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
3326  return C->isZero() && !C->isNegative();
3327  return false;
3328 }
3329 
3330 static unsigned inverseMinMax(unsigned Opc) {
3331  switch (Opc) {
3332  case ISD::FMAXNUM:
3333  return ISD::FMINNUM;
3334  case ISD::FMINNUM:
3335  return ISD::FMAXNUM;
3337  return AMDGPUISD::FMIN_LEGACY;
3339  return AMDGPUISD::FMAX_LEGACY;
3340  default:
3341  llvm_unreachable("invalid min/max opcode");
3342  }
3343 }
3344 
3346  DAGCombinerInfo &DCI) const {
3347  SelectionDAG &DAG = DCI.DAG;
3348  SDValue N0 = N->getOperand(0);
3349  EVT VT = N->getValueType(0);
3350 
3351  unsigned Opc = N0.getOpcode();
3352 
3353  // If the input has multiple uses and we can either fold the negate down, or
3354  // the other uses cannot, give up. This both prevents unprofitable
3355  // transformations and infinite loops: we won't repeatedly try to fold around
3356  // a negate that has no 'good' form.
3357  if (N0.hasOneUse()) {
3358  // This may be able to fold into the source, but at a code size cost. Don't
3359  // fold if the fold into the user is free.
3360  if (allUsesHaveSourceMods(N, 0))
3361  return SDValue();
3362  } else {
3363  if (fnegFoldsIntoOp(Opc) &&
3365  return SDValue();
3366  }
3367 
3368  SDLoc SL(N);
3369  switch (Opc) {
3370  case ISD::FADD: {
3371  if (!mayIgnoreSignedZero(N0))
3372  return SDValue();
3373 
3374  // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
3375  SDValue LHS = N0.getOperand(0);
3376  SDValue RHS = N0.getOperand(1);
3377 
3378  if (LHS.getOpcode() != ISD::FNEG)
3379  LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
3380  else
3381  LHS = LHS.getOperand(0);
3382 
3383  if (RHS.getOpcode() != ISD::FNEG)
3384  RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3385  else
3386  RHS = RHS.getOperand(0);
3387 
3388  SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
3389  if (!N0.hasOneUse())
3390  DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3391  return Res;
3392  }
3393  case ISD::FMUL:
3394  case AMDGPUISD::FMUL_LEGACY: {
3395  // (fneg (fmul x, y)) -> (fmul x, (fneg y))
3396  // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
3397  SDValue LHS = N0.getOperand(0);
3398  SDValue RHS = N0.getOperand(1);
3399 
3400  if (LHS.getOpcode() == ISD::FNEG)
3401  LHS = LHS.getOperand(0);
3402  else if (RHS.getOpcode() == ISD::FNEG)
3403  RHS = RHS.getOperand(0);
3404  else
3405  RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3406 
3407  SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
3408  if (!N0.hasOneUse())
3409  DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3410  return Res;
3411  }
3412  case ISD::FMA:
3413  case ISD::FMAD: {
3414  if (!mayIgnoreSignedZero(N0))
3415  return SDValue();
3416 
3417  // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
3418  SDValue LHS = N0.getOperand(0);
3419  SDValue MHS = N0.getOperand(1);
3420  SDValue RHS = N0.getOperand(2);
3421 
3422  if (LHS.getOpcode() == ISD::FNEG)
3423  LHS = LHS.getOperand(0);
3424  else if (MHS.getOpcode() == ISD::FNEG)
3425  MHS = MHS.getOperand(0);
3426  else
3427  MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
3428 
3429  if (RHS.getOpcode() != ISD::FNEG)
3430  RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3431  else
3432  RHS = RHS.getOperand(0);
3433 
3434  SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
3435  if (!N0.hasOneUse())
3436  DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3437  return Res;
3438  }
3439  case ISD::FMAXNUM:
3440  case ISD::FMINNUM:
3442  case AMDGPUISD::FMIN_LEGACY: {
3443  // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
3444  // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
3445  // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
3446  // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
3447 
3448  SDValue LHS = N0.getOperand(0);
3449  SDValue RHS = N0.getOperand(1);
3450 
3451  // 0 doesn't have a negated inline immediate.
3452  // TODO: Shouldn't fold 1/2pi either, and should be generalized to other
3453  // operations.
3454  if (isConstantFPZero(RHS))
3455  return SDValue();
3456 
3457  SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
3458  SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3459  unsigned Opposite = inverseMinMax(Opc);
3460 
3461  SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
3462  if (!N0.hasOneUse())
3463  DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3464  return Res;
3465  }
3466  case ISD::FP_EXTEND:
3467  case ISD::FTRUNC:
3468  case ISD::FRINT:
3469  case ISD::FNEARBYINT: // XXX - Should fround be handled?
3470  case ISD::FSIN:
3471  case AMDGPUISD::RCP:
3472  case AMDGPUISD::RCP_LEGACY:
3473  case AMDGPUISD::SIN_HW: {
3474  SDValue CvtSrc = N0.getOperand(0);
3475  if (CvtSrc.getOpcode() == ISD::FNEG) {
3476  // (fneg (fp_extend (fneg x))) -> (fp_extend x)
3477  // (fneg (rcp (fneg x))) -> (rcp x)
3478  return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
3479  }
3480 
3481  if (!N0.hasOneUse())
3482  return SDValue();
3483 
3484  // (fneg (fp_extend x)) -> (fp_extend (fneg x))
3485  // (fneg (rcp x)) -> (rcp (fneg x))
3486  SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3487  return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
3488  }
3489  case ISD::FP_ROUND: {
3490  SDValue CvtSrc = N0.getOperand(0);
3491 
3492  if (CvtSrc.getOpcode() == ISD::FNEG) {
3493  // (fneg (fp_round (fneg x))) -> (fp_round x)
3494  return DAG.getNode(ISD::FP_ROUND, SL, VT,
3495  CvtSrc.getOperand(0), N0.getOperand(1));
3496  }
3497 
3498  if (!N0.hasOneUse())
3499  return SDValue();
3500 
3501  // (fneg (fp_round x)) -> (fp_round (fneg x))
3502  SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3503  return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
3504  }
3505  case ISD::FP16_TO_FP: {
3506  // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
3507  // f16, but legalization of f16 fneg ends up pulling it out of the source.
3508  // Put the fneg back as a legal source operation that can be matched later.
3509  SDLoc SL(N);
3510 
3511  SDValue Src = N0.getOperand(0);
3512  EVT SrcVT = Src.getValueType();
3513 
3514  // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
3515  SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
3516  DAG.getConstant(0x8000, SL, SrcVT));
3517  return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg);
3518  }
3519  default:
3520  return SDValue();
3521  }
3522 }
3523 
3525  DAGCombinerInfo &DCI) const {
3526  SelectionDAG &DAG = DCI.DAG;
3527  SDValue N0 = N->getOperand(0);
3528 
3529  if (!N0.hasOneUse())
3530  return SDValue();
3531 
3532  switch (N0.getOpcode()) {
3533  case ISD::FP16_TO_FP: {
3534  assert(!Subtarget->has16BitInsts() && "should only see if f16 is illegal");
3535  SDLoc SL(N);
3536  SDValue Src = N0.getOperand(0);
3537  EVT SrcVT = Src.getValueType();
3538 
3539  // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
3540  SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
3541  DAG.getConstant(0x7fff, SL, SrcVT));
3542  return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
3543  }
3544  default:
3545  return SDValue();
3546  }
3547 }
3548 
3550  DAGCombinerInfo &DCI) const {
3551  SelectionDAG &DAG = DCI.DAG;
3552  SDLoc DL(N);
3553 
3554  switch(N->getOpcode()) {
3555  default:
3556  break;
3557  case ISD::BITCAST: {
3558  EVT DestVT = N->getValueType(0);
3559 
3560  // Push casts through vector builds. This helps avoid emitting a large
3561  // number of copies when materializing floating point vector constants.
3562  //
3563  // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
3564  // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
3565  if (DestVT.isVector()) {
3566  SDValue Src = N->getOperand(0);
3567  if (Src.getOpcode() == ISD::BUILD_VECTOR) {
3568  EVT SrcVT = Src.getValueType();
3569  unsigned NElts = DestVT.getVectorNumElements();
3570 
3571  if (SrcVT.getVectorNumElements() == NElts) {
3572  EVT DestEltVT = DestVT.getVectorElementType();
3573 
3574  SmallVector<SDValue, 8> CastedElts;
3575  SDLoc SL(N);
3576  for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
3577  SDValue Elt = Src.getOperand(I);
3578  CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt));
3579  }
3580 
3581  return DAG.getBuildVector(DestVT, SL, CastedElts);
3582  }
3583  }
3584  }
3585 
3586  if (DestVT.getSizeInBits() != 64 && !DestVT.isVector())
3587  break;
3588 
3589  // Fold bitcasts of constants.
3590  //
3591  // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
3592  // TODO: Generalize and move to DAGCombiner
3593  SDValue Src = N->getOperand(0);
3594  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src)) {
3595  assert(Src.getValueType() == MVT::i64);
3596  SDLoc SL(N);
3597  uint64_t CVal = C->getZExtValue();
3598  return DAG.getNode(ISD::BUILD_VECTOR, SL, DestVT,
3599  DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
3600  DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
3601  }
3602 
3603  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Src)) {
3604  const APInt &Val = C->getValueAPF().bitcastToAPInt();
3605  SDLoc SL(N);
3606  uint64_t CVal = Val.getZExtValue();
3608  DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
3609  DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
3610 
3611  return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec);
3612  }
3613 
3614  break;
3615  }
3616  case ISD::SHL: {
3618  break;
3619 
3620  return performShlCombine(N, DCI);
3621  }
3622  case ISD::SRL: {
3624  break;
3625 
3626  return performSrlCombine(N, DCI);
3627  }
3628  case ISD::SRA: {
3630  break;
3631 
3632  return performSraCombine(N, DCI);
3633  }
3634  case ISD::MUL:
3635  return performMulCombine(N, DCI);
3636  case ISD::MULHS:
3637  return performMulhsCombine(N, DCI);
3638  case ISD::MULHU:
3639  return performMulhuCombine(N, DCI);
3640  case AMDGPUISD::MUL_I24:
3641  case AMDGPUISD::MUL_U24:
3642  case AMDGPUISD::MULHI_I24:
3643  case AMDGPUISD::MULHI_U24: {
3644  // If the first call to simplify is successfull, then N may end up being
3645  // deleted, so we shouldn't call simplifyI24 again.
3646  simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI);
3647  return SDValue();
3648  }
3651  return performMulLoHi24Combine(N, DCI);
3652  case ISD::SELECT:
3653  return performSelectCombine(N, DCI);
3654  case ISD::FNEG:
3655  return performFNegCombine(N, DCI);
3656  case ISD::FABS:
3657  return performFAbsCombine(N, DCI);
3658  case AMDGPUISD::BFE_I32:
3659  case AMDGPUISD::BFE_U32: {
3660  assert(!N->getValueType(0).isVector() &&
3661  "Vector handling of BFE not implemented");
3663  if (!Width)
3664  break;
3665 
3666  uint32_t WidthVal = Width->getZExtValue() & 0x1f;
3667  if (WidthVal == 0)
3668  return DAG.getConstant(0, DL, MVT::i32);
3669 
3671  if (!Offset)
3672  break;
3673 
3674  SDValue BitsFrom = N->getOperand(0);
3675  uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
3676 
3677  bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
3678 
3679  if (OffsetVal == 0) {
3680  // This is already sign / zero extended, so try to fold away extra BFEs.
3681  unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
3682 
3683  unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
3684  if (OpSignBits >= SignBits)
3685  return BitsFrom;
3686 
3687  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
3688  if (Signed) {
3689  // This is a sign_extend_inreg. Replace it to take advantage of existing
3690  // DAG Combines. If not eliminated, we will match back to BFE during
3691  // selection.
3692 
3693  // TODO: The sext_inreg of extended types ends, although we can could
3694  // handle them in a single BFE.
3695  return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
3696  DAG.getValueType(SmallVT));
3697  }
3698 
3699  return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
3700  }
3701 
3702  if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) {
3703  if (Signed) {
3704  return constantFoldBFE<int32_t>(DAG,
3705  CVal->getSExtValue(),
3706  OffsetVal,
3707  WidthVal,
3708  DL);
3709  }
3710 
3711  return constantFoldBFE<uint32_t>(DAG,
3712  CVal->getZExtValue(),
3713  OffsetVal,
3714  WidthVal,
3715  DL);
3716  }
3717 
3718  if ((OffsetVal + WidthVal) >= 32 &&
3719  !(Subtarget->hasSDWA() && OffsetVal == 16 && WidthVal == 16)) {
3720  SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
3721  return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
3722  BitsFrom, ShiftVal);
3723  }
3724 
3725  if (BitsFrom.hasOneUse()) {
3726  APInt Demanded = APInt::getBitsSet(32,
3727  OffsetVal,
3728  OffsetVal + WidthVal);
3729 
3730  KnownBits Known;
3732  !DCI.isBeforeLegalizeOps());
3733  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3734  if (TLI.ShrinkDemandedConstant(BitsFrom, Demanded, TLO) ||
3735  TLI.SimplifyDemandedBits(BitsFrom, Demanded, Known, TLO)) {
3736  DCI.CommitTargetLoweringOpt(TLO);
3737  }
3738  }
3739 
3740  break;
3741  }
3742  case ISD::LOAD:
3743  return performLoadCombine(N, DCI);
3744  case ISD::STORE:
3745  return performStoreCombine(N, DCI);
3746  case AMDGPUISD::CLAMP:
3747  return performClampCombine(N, DCI);
3748  case AMDGPUISD::RCP: {
3749  if (const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
3750  // XXX - Should this flush denormals?
3751  const APFloat &Val = CFP->getValueAPF();
3752  APFloat One(Val.getSemantics(), "1.0");
3753  return DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
3754  }
3755 
3756  break;
3757  }
3758  case ISD::AssertZext:
3759  case ISD::AssertSext:
3760  return performAssertSZExtCombine(N, DCI);
3761  }
3762  return SDValue();
3763 }
3764 
3765 //===----------------------------------------------------------------------===//
3766 // Helper functions
3767 //===----------------------------------------------------------------------===//
3768 
3770  const TargetRegisterClass *RC,
3771  unsigned Reg, EVT VT,
3772  const SDLoc &SL,
3773  bool RawReg) const {
3774  MachineFunction &MF = DAG.getMachineFunction();
3776  unsigned VReg;
3777 
3778  if (!MRI.isLiveIn(Reg)) {
3779  VReg = MRI.createVirtualRegister(RC);
3780  MRI.addLiveIn(Reg, VReg);
3781  } else {
3782  VReg = MRI.getLiveInVirtReg(Reg);
3783  }
3784 
3785  if (RawReg)
3786  return DAG.getRegister(VReg, VT);
3787 
3788  return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT);
3789 }
3790 
3792  EVT VT,
3793  const SDLoc &SL,
3794  int64_t Offset) const {
3795  MachineFunction &MF = DAG.getMachineFunction();
3796  MachineFrameInfo &MFI = MF.getFrameInfo();
3797 
3798  int FI = MFI.CreateFixedObject(VT.getStoreSize(), Offset, true);
3799  auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
3800  SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);
3801 
3802  return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, 4,
3805 }
3806 
3808  const SDLoc &SL,
3809  SDValue Chain,
3810  SDValue StackPtr,
3811  SDValue ArgVal,
3812  int64_t Offset) const {
3813  MachineFunction &MF = DAG.getMachineFunction();
3814  MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
3815  SDValue PtrOffset = DAG.getConstant(Offset, SL, MVT::i32);
3816  SDValue Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, StackPtr, PtrOffset);
3817 
3818  SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, 4,
3820  return Store;
3821 }
3822 
3824  const TargetRegisterClass *RC,
3825  EVT VT, const SDLoc &SL,
3826  const ArgDescriptor &Arg) const {
3827  assert(Arg && "Attempting to load missing argument");
3828 
3829  if (Arg.isRegister())
3830  return CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL);
3831  return loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
3832 }
3833 
3835  const AMDGPUMachineFunction *MFI, const ImplicitParameter Param) const {
3836  unsigned Alignment = Subtarget->getAlignmentForImplicitArgPtr();
3837  uint64_t ArgOffset = alignTo(MFI->getABIArgOffset(), Alignment);
3838  switch (Param) {
3839  case GRID_DIM:
3840  return ArgOffset;
3841  case GRID_OFFSET:
3842  return ArgOffset + 4;
3843  }
3844  llvm_unreachable("unexpected implicit parameter type");
3845 }
3846 
3847 #define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
3848 
3849 const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
3850  switch ((AMDGPUISD::NodeType)Opcode) {
3851  case AMDGPUISD::FIRST_NUMBER: break;
3852  // AMDIL DAG nodes
3855 
3856  // AMDGPU DAG nodes
3975  }
3976  return nullptr;
3977 }
3978 
3980  SelectionDAG &DAG, int Enabled,
3981  int &RefinementSteps,
3982  bool &UseOneConstNR,
3983  bool Reciprocal) const {
3984  EVT VT = Operand.getValueType();
3985 
3986  if (VT == MVT::f32) {
3987  RefinementSteps = 0;
3988  return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
3989  }
3990 
3991  // TODO: There is also f64 rsq instruction, but the documentation is less
3992  // clear on its precision.
3993 
3994  return SDValue();
3995 }
3996 
3998  SelectionDAG &DAG, int Enabled,
3999  int &RefinementSteps) const {
4000  EVT VT = Operand.getValueType();
4001 
4002  if (VT == MVT::f32) {
4003  // Reciprocal, < 1 ulp error.
4004  //
4005  // This reciprocal approximation converges to < 0.5 ulp error with one
4006  // newton rhapson performed with two fused multiple adds (FMAs).
4007 
4008  RefinementSteps = 0;
4009  return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);
4010  }
4011 
4012  // TODO: There is also f64 rcp instruction, but the documentation is less
4013  // clear on its precision.
4014 
4015  return SDValue();
4016 }
4017 
4019  const SDValue Op, KnownBits &Known,
4020  const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
4021 
4022  Known.resetAll(); // Don't know anything.
4023 
4024  unsigned Opc = Op.getOpcode();
4025 
4026  switch (Opc) {
4027  default:
4028  break;
4029  case AMDGPUISD::CARRY:
4030  case AMDGPUISD::BORROW: {
4031  Known.Zero = APInt::getHighBitsSet(32, 31);
4032  break;
4033  }
4034 
4035  case AMDGPUISD::BFE_I32:
4036  case AMDGPUISD::BFE_U32: {
4038  if (!CWidth)
4039  return;
4040 
4041  uint32_t Width = CWidth->getZExtValue() & 0x1f;
4042 
4043  if (Opc == AMDGPUISD::BFE_U32)
4044  Known.Zero = APInt::getHighBitsSet(32, 32 - Width);
4045 
4046  break;
4047  }
4048  case AMDGPUISD::FP_TO_FP16:
4049  case AMDGPUISD::FP16_ZEXT: {
4050  unsigned BitWidth = Known.getBitWidth();
4051 
4052  // High bits are zero.
4053  Known.Zero = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
4054  break;
4055  }
4056  case AMDGPUISD::MUL_U24:
4057  case AMDGPUISD::MUL_I24: {
4058  KnownBits LHSKnown, RHSKnown;
4059  DAG.computeKnownBits(Op.getOperand(0), LHSKnown, Depth + 1);
4060  DAG.computeKnownBits(Op.getOperand(1), RHSKnown, Depth + 1);
4061 
4062  unsigned TrailZ = LHSKnown.countMinTrailingZeros() +
4063  RHSKnown.countMinTrailingZeros();
4064  Known.Zero.setLowBits(std::min(TrailZ, 32u));
4065 
4066  unsigned LHSValBits = 32 - std::max(LHSKnown.countMinSignBits(), 8u);
4067  unsigned RHSValBits = 32 - std::max(RHSKnown.countMinSignBits(), 8u);
4068  unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
4069  if (MaxValBits >= 32)
4070  break;
4071  bool Negative = false;
4072  if (Opc == AMDGPUISD::MUL_I24) {
4073  bool LHSNegative = !!(LHSKnown.One & (1 << 23));
4074  bool LHSPositive = !!(LHSKnown.Zero & (1 << 23));
4075  bool RHSNegative = !!(RHSKnown.One & (1 << 23));
4076  bool RHSPositive = !!(RHSKnown.Zero & (1 << 23));
4077  if ((!LHSNegative && !LHSPositive) || (!RHSNegative && !RHSPositive))
4078  break;
4079  Negative = (LHSNegative && RHSPositive) || (LHSPositive && RHSNegative);
4080  }
4081  if (Negative)
4082  Known.One.setHighBits(32 - MaxValBits);
4083  else
4084  Known.Zero.setHighBits(32 - MaxValBits);
4085  break;