LLVM  9.0.0svn
SystemZISelLowering.cpp
Go to the documentation of this file.
1 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SystemZTargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SystemZISelLowering.h"
14 #include "SystemZCallingConv.h"
17 #include "SystemZTargetMachine.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/Support/KnownBits.h"
26 #include <cctype>
27 
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "systemz-lower"
31 
32 namespace {
33 // Represents information about a comparison.
34 struct Comparison {
35  Comparison(SDValue Op0In, SDValue Op1In)
36  : Op0(Op0In), Op1(Op1In), Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
37 
38  // The operands to the comparison.
39  SDValue Op0, Op1;
40 
41  // The opcode that should be used to compare Op0 and Op1.
42  unsigned Opcode;
43 
44  // A SystemZICMP value. Only used for integer comparisons.
45  unsigned ICmpType;
46 
47  // The mask of CC values that Opcode can produce.
48  unsigned CCValid;
49 
50  // The mask of CC values for which the original condition is true.
51  unsigned CCMask;
52 };
53 } // end anonymous namespace
54 
55 // Classify VT as either 32 or 64 bit.
56 static bool is32Bit(EVT VT) {
57  switch (VT.getSimpleVT().SimpleTy) {
58  case MVT::i32:
59  return true;
60  case MVT::i64:
61  return false;
62  default:
63  llvm_unreachable("Unsupported type");
64  }
65 }
66 
67 // Return a version of MachineOperand that can be safely used before the
68 // final use.
70  if (Op.isReg())
71  Op.setIsKill(false);
72  return Op;
73 }
74 
76  const SystemZSubtarget &STI)
77  : TargetLowering(TM), Subtarget(STI) {
78  MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize(0));
79 
80  // Set up the register classes.
81  if (Subtarget.hasHighWord())
82  addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass);
83  else
84  addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass);
85  addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
86  if (Subtarget.hasVector()) {
87  addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
88  addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
89  } else {
90  addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
91  addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
92  }
93  if (Subtarget.hasVectorEnhancements1())
94  addRegisterClass(MVT::f128, &SystemZ::VR128BitRegClass);
95  else
96  addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass);
97 
98  if (Subtarget.hasVector()) {
99  addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass);
100  addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass);
101  addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass);
102  addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass);
103  addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass);
104  addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass);
105  }
106 
107  // Compute derived properties from the register classes
109 
110  // Set up special registers.
112 
113  // TODO: It may be better to default to latency-oriented scheduling, however
114  // LLVM's current latency-oriented scheduler can't handle physreg definitions
115  // such as SystemZ has with CC, so set this to the register-pressure
116  // scheduler, because it can.
118 
121 
122  // Instructions are strings of 2-byte aligned 2-byte values.
124  // For performance reasons we prefer 16-byte alignment.
126 
127  // Handle operations that are handled in a similar way for all types.
128  for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
130  ++I) {
131  MVT VT = MVT::SimpleValueType(I);
132  if (isTypeLegal(VT)) {
133  // Lower SET_CC into an IPM-based sequence.
135 
136  // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
138 
139  // Lower SELECT_CC and BR_CC into separate comparisons and branches.
142  }
143  }
144 
145  // Expand jump table branches as address arithmetic followed by an
146  // indirect jump.
148 
149  // Expand BRCOND into a BR_CC (see above).
151 
152  // Handle integer types.
153  for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
155  ++I) {
156  MVT VT = MVT::SimpleValueType(I);
157  if (isTypeLegal(VT)) {
158  // Expand individual DIV and REMs into DIVREMs.
165 
166  // Support addition/subtraction with overflow.
169 
170  // Support addition/subtraction with carry.
173 
174  // Support carry in as value rather than glue.
177 
178  // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and
179  // stores, putting a serialization instruction after the stores.
182 
183  // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
184  // available, or if the operand is constant.
186 
187  // Use POPCNT on z196 and above.
188  if (Subtarget.hasPopulationCount())
190  else
192 
193  // No special instructions for these.
196 
197  // Use *MUL_LOHI where possible instead of MULH*.
202 
203  // Only z196 and above have native support for conversions to unsigned.
204  // On z10, promoting to i64 doesn't generate an inexact condition for
205  // values that are outside the i32 range but in the i64 range, so use
206  // the default expansion.
207  if (!Subtarget.hasFPExtension())
209  }
210  }
211 
212  // Type legalization will convert 8- and 16-bit atomic operations into
213  // forms that operate on i32s (but still keeping the original memory VT).
214  // Lower them into full i32 operations.
226 
227  // Even though i128 is not a legal type, we still need to custom lower
228  // the atomic operations in order to exploit SystemZ instructions.
231 
232  // We can use the CC result of compare-and-swap to implement
233  // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
237 
239 
240  // Traps are legal, as we will convert them to "j .+2".
242 
243  // z10 has instructions for signed but not unsigned FP conversion.
244  // Handle unsigned 32-bit types as signed 64-bit types.
245  if (!Subtarget.hasFPExtension()) {
248  }
249 
250  // We have native support for a 64-bit CTLZ, via FLOGR.
254 
255  // Give LowerOperation the chance to replace 64-bit ORs with subregs.
257 
258  // FIXME: Can we support these natively?
262 
263  // We have native instructions for i8, i16 and i32 extensions, but not i1.
265  for (MVT VT : MVT::integer_valuetypes()) {
269  }
270 
271  // Handle the various types of symbolic address.
277 
278  // We need to handle dynamic allocations specially because of the
279  // 160-byte area at the bottom of the stack.
282 
283  // Use custom expanders so that we can force the function to use
284  // a frame pointer.
287 
288  // Handle prefetches with PFD or PFDRL.
290 
291  for (MVT VT : MVT::vector_valuetypes()) {
292  // Assume by default that all vector operations need to be expanded.
293  for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
294  if (getOperationAction(Opcode, VT) == Legal)
295  setOperationAction(Opcode, VT, Expand);
296 
297  // Likewise all truncating stores and extending loads.
298  for (MVT InnerVT : MVT::vector_valuetypes()) {
299  setTruncStoreAction(VT, InnerVT, Expand);
300  setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
301  setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
302  setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
303  }
304 
305  if (isTypeLegal(VT)) {
306  // These operations are legal for anything that can be stored in a
307  // vector register, even if there is no native support for the format
308  // as such. In particular, we can do these for v4f32 even though there
309  // are no specific instructions for that format.
315 
316  // Likewise, except that we need to replace the nodes with something
317  // more specific.
320  }
321  }
322 
323  // Handle integer vector types.
324  for (MVT VT : MVT::integer_vector_valuetypes()) {
325  if (isTypeLegal(VT)) {
326  // These operations have direct equivalents.
331  if (VT != MVT::v2i64)
336  if (Subtarget.hasVectorEnhancements1())
338  else
342 
343  // Convert a GPR scalar to a vector by inserting it into element 0.
345 
346  // Use a series of unpacks for extensions.
349 
350  // Detect shifts by a scalar amount and convert them into
351  // V*_BY_SCALAR.
355 
356  // At present ROTL isn't matched by DAGCombiner. ROTR should be
357  // converted into ROTL.
360 
361  // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
362  // and inverting the result as necessary.
364  }
365  }
366 
367  if (Subtarget.hasVector()) {
368  // There should be no need to check for float types other than v2f64
369  // since <2 x f32> isn't a legal type.
378  }
379 
380  // Handle floating-point types.
381  for (unsigned I = MVT::FIRST_FP_VALUETYPE;
383  ++I) {
384  MVT VT = MVT::SimpleValueType(I);
385  if (isTypeLegal(VT)) {
386  // We can use FI for FRINT.
388 
389  // We can use the extended form of FI for other rounding operations.
390  if (Subtarget.hasFPExtension()) {
396  }
397 
398  // No special instructions for these.
404 
405  // Handle constrained floating-point operations.
415  if (Subtarget.hasFPExtension()) {
421  }
422  }
423  }
424 
425  // Handle floating-point vector types.
426  if (Subtarget.hasVector()) {
427  // Scalar-to-vector conversion is just a subreg.
430 
431  // Some insertions and extractions can be done directly but others
432  // need to go via integers.
437 
438  // These operations have direct equivalents.
453 
454  // Handle constrained floating-point operations.
467  }
468 
469  // The vector enhancements facility 1 has instructions for these.
470  if (Subtarget.hasVectorEnhancements1()) {
485 
490 
495 
500 
505 
510 
511  // Handle constrained floating-point operations.
524  for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
525  MVT::v4f32, MVT::v2f64 }) {
528  }
529  }
530 
531  // We have fused multiply-addition for f32 and f64 but not f128.
534  if (Subtarget.hasVectorEnhancements1())
536  else
538 
539  // We don't have a copysign instruction on vector registers.
540  if (Subtarget.hasVectorEnhancements1())
542 
543  // Needed so that we don't try to implement f128 constant loads using
544  // a load-and-extend of a f80 constant (in cases where the constant
545  // would fit in an f80).
546  for (MVT VT : MVT::fp_valuetypes())
548 
549  // We don't have extending load instruction on vector registers.
550  if (Subtarget.hasVectorEnhancements1()) {
553  }
554 
555  // Floating-point truncation and stores need to be done separately.
559 
560  // We have 64-bit FPR<->GPR moves, but need special handling for
561  // 32-bit forms.
562  if (!Subtarget.hasVector()) {
565  }
566 
567  // VASTART and VACOPY need to deal with the SystemZ-specific varargs
568  // structure, but VAEND is a no-op.
572 
573  // Codes for which we want to perform some z-specific combinations.
587 
588  // Handle intrinsics.
591 
592  // We want to use MVC in preference to even a single load/store pair.
593  MaxStoresPerMemcpy = 0;
595 
596  // The main memset sequence is a byte store followed by an MVC.
597  // Two STC or MV..I stores win over that, but the kind of fused stores
598  // generated by target-independent code don't when the byte value is
599  // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
600  // than "STC;MVC". Handle the choice in target-specific code instead.
601  MaxStoresPerMemset = 0;
603 }
604 
606  LLVMContext &, EVT VT) const {
607  if (!VT.isVector())
608  return MVT::i32;
610 }
611 
613  VT = VT.getScalarType();
614 
615  if (!VT.isSimple())
616  return false;
617 
618  switch (VT.getSimpleVT().SimpleTy) {
619  case MVT::f32:
620  case MVT::f64:
621  return true;
622  case MVT::f128:
623  return Subtarget.hasVectorEnhancements1();
624  default:
625  break;
626  }
627 
628  return false;
629 }
630 
631 // Return true if the constant can be generated with a vector instruction,
632 // such as VGM, VGMB or VREPI.
634  const SystemZSubtarget &Subtarget) {
635  const SystemZInstrInfo *TII =
636  static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
637  if (!Subtarget.hasVector() ||
638  (isFP128 && !Subtarget.hasVectorEnhancements1()))
639  return false;
640 
641  // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
642  // preferred way of creating all-zero and all-one vectors so give it
643  // priority over other methods below.
644  unsigned Mask = 0;
645  unsigned I = 0;
646  for (; I < SystemZ::VectorBytes; ++I) {
647  uint64_t Byte = IntBits.lshr(I * 8).trunc(8).getZExtValue();
648  if (Byte == 0xff)
649  Mask |= 1ULL << I;
650  else if (Byte != 0)
651  break;
652  }
653  if (I == SystemZ::VectorBytes) {
654  Opcode = SystemZISD::BYTE_MASK;
655  OpVals.push_back(Mask);
656  VecVT = MVT::getVectorVT(MVT::getIntegerVT(8), 16);
657  return true;
658  }
659 
660  if (SplatBitSize > 64)
661  return false;
662 
663  auto tryValue = [&](uint64_t Value) -> bool {
664  // Try VECTOR REPLICATE IMMEDIATE
665  int64_t SignedValue = SignExtend64(Value, SplatBitSize);
666  if (isInt<16>(SignedValue)) {
667  OpVals.push_back(((unsigned) SignedValue));
668  Opcode = SystemZISD::REPLICATE;
669  VecVT = MVT::getVectorVT(MVT::getIntegerVT(SplatBitSize),
670  SystemZ::VectorBits / SplatBitSize);
671  return true;
672  }
673  // Try VECTOR GENERATE MASK
674  unsigned Start, End;
675  if (TII->isRxSBGMask(Value, SplatBitSize, Start, End)) {
676  // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
677  // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
678  // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
679  OpVals.push_back(Start - (64 - SplatBitSize));
680  OpVals.push_back(End - (64 - SplatBitSize));
681  Opcode = SystemZISD::ROTATE_MASK;
682  VecVT = MVT::getVectorVT(MVT::getIntegerVT(SplatBitSize),
683  SystemZ::VectorBits / SplatBitSize);
684  return true;
685  }
686  return false;
687  };
688 
689  // First try assuming that any undefined bits above the highest set bit
690  // and below the lowest set bit are 1s. This increases the likelihood of
691  // being able to use a sign-extended element value in VECTOR REPLICATE
692  // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
693  uint64_t SplatBitsZ = SplatBits.getZExtValue();
694  uint64_t SplatUndefZ = SplatUndef.getZExtValue();
695  uint64_t Lower =
696  (SplatUndefZ & ((uint64_t(1) << findFirstSet(SplatBitsZ)) - 1));
697  uint64_t Upper =
698  (SplatUndefZ & ~((uint64_t(1) << findLastSet(SplatBitsZ)) - 1));
699  if (tryValue(SplatBitsZ | Upper | Lower))
700  return true;
701 
702  // Now try assuming that any undefined bits between the first and
703  // last defined set bits are set. This increases the chances of
704  // using a non-wraparound mask.
705  uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
706  return tryValue(SplatBitsZ | Middle);
707 }
708 
710  IntBits = FPImm.bitcastToAPInt().zextOrSelf(128);
711  isFP128 = (&FPImm.getSemantics() == &APFloat::IEEEquad());
712 
713  // Find the smallest splat.
714  SplatBits = FPImm.bitcastToAPInt();
715  unsigned Width = SplatBits.getBitWidth();
716  while (Width > 8) {
717  unsigned HalfSize = Width / 2;
718  APInt HighValue = SplatBits.lshr(HalfSize).trunc(HalfSize);
719  APInt LowValue = SplatBits.trunc(HalfSize);
720 
721  // If the two halves do not match, stop here.
722  if (HighValue != LowValue || 8 > HalfSize)
723  break;
724 
725  SplatBits = HighValue;
726  Width = HalfSize;
727  }
728  SplatUndef = 0;
729  SplatBitSize = Width;
730 }
731 
733  assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
734  bool HasAnyUndefs;
735 
736  // Get IntBits by finding the 128 bit splat.
737  BVN->isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
738  true);
739 
740  // Get SplatBits by finding the 8 bit or greater splat.
741  BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
742  true);
743 }
744 
746  bool ForCodeSize) const {
747  // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
748  if (Imm.isZero() || Imm.isNegZero())
749  return true;
750 
751  return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
752 }
753 
755  // We can use CGFI or CLGFI.
756  return isInt<32>(Imm) || isUInt<32>(Imm);
757 }
758 
760  // We can use ALGFI or SLGFI.
761  return isUInt<32>(Imm) || isUInt<32>(-Imm);
762 }
763 
765  EVT VT, unsigned, unsigned, MachineMemOperand::Flags, bool *Fast) const {
766  // Unaligned accesses should never be slower than the expanded version.
767  // We check specifically for aligned accesses in the few cases where
768  // they are required.
769  if (Fast)
770  *Fast = true;
771  return true;
772 }
773 
774 // Information about the addressing mode for a memory access.
776  // True if a long displacement is supported.
778 
779  // True if use of index register is supported.
780  bool IndexReg;
781 
782  AddressingMode(bool LongDispl, bool IdxReg) :
783  LongDisplacement(LongDispl), IndexReg(IdxReg) {}
784 };
785 
786 // Return the desired addressing mode for a Load which has only one use (in
787 // the same block) which is a Store.
788 static AddressingMode getLoadStoreAddrMode(bool HasVector,
789  Type *Ty) {
790  // With vector support a Load->Store combination may be combined to either
791  // an MVC or vector operations and it seems to work best to allow the
792  // vector addressing mode.
793  if (HasVector)
794  return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
795 
796  // Otherwise only the MVC case is special.
797  bool MVC = Ty->isIntegerTy(8);
798  return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
799 }
800 
801 // Return the addressing mode which seems most desirable given an LLVM
802 // Instruction pointer.
803 static AddressingMode
805  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
806  switch (II->getIntrinsicID()) {
807  default: break;
808  case Intrinsic::memset:
809  case Intrinsic::memmove:
810  case Intrinsic::memcpy:
811  return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
812  }
813  }
814 
815  if (isa<LoadInst>(I) && I->hasOneUse()) {
816  auto *SingleUser = dyn_cast<Instruction>(*I->user_begin());
817  if (SingleUser->getParent() == I->getParent()) {
818  if (isa<ICmpInst>(SingleUser)) {
819  if (auto *C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
820  if (C->getBitWidth() <= 64 &&
821  (isInt<16>(C->getSExtValue()) || isUInt<16>(C->getZExtValue())))
822  // Comparison of memory with 16 bit signed / unsigned immediate
823  return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
824  } else if (isa<StoreInst>(SingleUser))
825  // Load->Store
826  return getLoadStoreAddrMode(HasVector, I->getType());
827  }
828  } else if (auto *StoreI = dyn_cast<StoreInst>(I)) {
829  if (auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
830  if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
831  // Load->Store
832  return getLoadStoreAddrMode(HasVector, LoadI->getType());
833  }
834 
835  if (HasVector && (isa<LoadInst>(I) || isa<StoreInst>(I))) {
836 
837  // * Use LDE instead of LE/LEY for z13 to avoid partial register
838  // dependencies (LDE only supports small offsets).
839  // * Utilize the vector registers to hold floating point
840  // values (vector load / store instructions only support small
841  // offsets).
842 
843  Type *MemAccessTy = (isa<LoadInst>(I) ? I->getType() :
844  I->getOperand(0)->getType());
845  bool IsFPAccess = MemAccessTy->isFloatingPointTy();
846  bool IsVectorAccess = MemAccessTy->isVectorTy();
847 
848  // A store of an extracted vector element will be combined into a VSTE type
849  // instruction.
850  if (!IsVectorAccess && isa<StoreInst>(I)) {
851  Value *DataOp = I->getOperand(0);
852  if (isa<ExtractElementInst>(DataOp))
853  IsVectorAccess = true;
854  }
855 
856  // A load which gets inserted into a vector element will be combined into a
857  // VLE type instruction.
858  if (!IsVectorAccess && isa<LoadInst>(I) && I->hasOneUse()) {
859  User *LoadUser = *I->user_begin();
860  if (isa<InsertElementInst>(LoadUser))
861  IsVectorAccess = true;
862  }
863 
864  if (IsFPAccess || IsVectorAccess)
865  return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
866  }
867 
868  return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
869 }
870 
872  const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
873  // Punt on globals for now, although they can be used in limited
874  // RELATIVE LONG cases.
875  if (AM.BaseGV)
876  return false;
877 
878  // Require a 20-bit signed offset.
879  if (!isInt<20>(AM.BaseOffs))
880  return false;
881 
882  AddressingMode SupportedAM(true, true);
883  if (I != nullptr)
884  SupportedAM = supportedAddressingMode(I, Subtarget.hasVector());
885 
886  if (!SupportedAM.LongDisplacement && !isUInt<12>(AM.BaseOffs))
887  return false;
888 
889  if (!SupportedAM.IndexReg)
890  // No indexing allowed.
891  return AM.Scale == 0;
892  else
893  // Indexing is OK but no scale factor can be applied.
894  return AM.Scale == 0 || AM.Scale == 1;
895 }
896 
898  if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
899  return false;
900  unsigned FromBits = FromType->getPrimitiveSizeInBits();
901  unsigned ToBits = ToType->getPrimitiveSizeInBits();
902  return FromBits > ToBits;
903 }
904 
906  if (!FromVT.isInteger() || !ToVT.isInteger())
907  return false;
908  unsigned FromBits = FromVT.getSizeInBits();
909  unsigned ToBits = ToVT.getSizeInBits();
910  return FromBits > ToBits;
911 }
912 
913 //===----------------------------------------------------------------------===//
914 // Inline asm support
915 //===----------------------------------------------------------------------===//
916 
919  if (Constraint.size() == 1) {
920  switch (Constraint[0]) {
921  case 'a': // Address register
922  case 'd': // Data register (equivalent to 'r')
923  case 'f': // Floating-point register
924  case 'h': // High-part register
925  case 'r': // General-purpose register
926  case 'v': // Vector register
927  return C_RegisterClass;
928 
929  case 'Q': // Memory with base and unsigned 12-bit displacement
930  case 'R': // Likewise, plus an index
931  case 'S': // Memory with base and signed 20-bit displacement
932  case 'T': // Likewise, plus an index
933  case 'm': // Equivalent to 'T'.
934  return C_Memory;
935 
936  case 'I': // Unsigned 8-bit constant
937  case 'J': // Unsigned 12-bit constant
938  case 'K': // Signed 16-bit constant
939  case 'L': // Signed 20-bit displacement (on all targets we support)
940  case 'M': // 0x7fffffff
941  return C_Other;
942 
943  default:
944  break;
945  }
946  }
947  return TargetLowering::getConstraintType(Constraint);
948 }
949 
952  const char *constraint) const {
953  ConstraintWeight weight = CW_Invalid;
954  Value *CallOperandVal = info.CallOperandVal;
955  // If we don't have a value, we can't do a match,
956  // but allow it at the lowest weight.
957  if (!CallOperandVal)
958  return CW_Default;
959  Type *type = CallOperandVal->getType();
960  // Look at the constraint type.
961  switch (*constraint) {
962  default:
963  weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
964  break;
965 
966  case 'a': // Address register
967  case 'd': // Data register (equivalent to 'r')
968  case 'h': // High-part register
969  case 'r': // General-purpose register
970  if (CallOperandVal->getType()->isIntegerTy())
971  weight = CW_Register;
972  break;
973 
974  case 'f': // Floating-point register
975  if (type->isFloatingPointTy())
976  weight = CW_Register;
977  break;
978 
979  case 'v': // Vector register
980  if ((type->isVectorTy() || type->isFloatingPointTy()) &&
981  Subtarget.hasVector())
982  weight = CW_Register;
983  break;
984 
985  case 'I': // Unsigned 8-bit constant
986  if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
987  if (isUInt<8>(C->getZExtValue()))
988  weight = CW_Constant;
989  break;
990 
991  case 'J': // Unsigned 12-bit constant
992  if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
993  if (isUInt<12>(C->getZExtValue()))
994  weight = CW_Constant;
995  break;
996 
997  case 'K': // Signed 16-bit constant
998  if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
999  if (isInt<16>(C->getSExtValue()))
1000  weight = CW_Constant;
1001  break;
1002 
1003  case 'L': // Signed 20-bit displacement (on all targets we support)
1004  if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1005  if (isInt<20>(C->getSExtValue()))
1006  weight = CW_Constant;
1007  break;
1008 
1009  case 'M': // 0x7fffffff
1010  if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1011  if (C->getZExtValue() == 0x7fffffff)
1012  weight = CW_Constant;
1013  break;
1014  }
1015  return weight;
1016 }
1017 
1018 // Parse a "{tNNN}" register constraint for which the register type "t"
1019 // has already been verified. MC is the class associated with "t" and
1020 // Map maps 0-based register numbers to LLVM register numbers.
1021 static std::pair<unsigned, const TargetRegisterClass *>
1023  const unsigned *Map, unsigned Size) {
1024  assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1025  if (isdigit(Constraint[2])) {
1026  unsigned Index;
1027  bool Failed =
1028  Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index);
1029  if (!Failed && Index < Size && Map[Index])
1030  return std::make_pair(Map[Index], RC);
1031  }
1032  return std::make_pair(0U, nullptr);
1033 }
1034 
1035 std::pair<unsigned, const TargetRegisterClass *>
1037  const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1038  if (Constraint.size() == 1) {
1039  // GCC Constraint Letters
1040  switch (Constraint[0]) {
1041  default: break;
1042  case 'd': // Data register (equivalent to 'r')
1043  case 'r': // General-purpose register
1044  if (VT == MVT::i64)
1045  return std::make_pair(0U, &SystemZ::GR64BitRegClass);
1046  else if (VT == MVT::i128)
1047  return std::make_pair(0U, &SystemZ::GR128BitRegClass);
1048  return std::make_pair(0U, &SystemZ::GR32BitRegClass);
1049 
1050  case 'a': // Address register
1051  if (VT == MVT::i64)
1052  return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
1053  else if (VT == MVT::i128)
1054  return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
1055  return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
1056 
1057  case 'h': // High-part register (an LLVM extension)
1058  return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
1059 
1060  case 'f': // Floating-point register
1061  if (VT == MVT::f64)
1062  return std::make_pair(0U, &SystemZ::FP64BitRegClass);
1063  else if (VT == MVT::f128)
1064  return std::make_pair(0U, &SystemZ::FP128BitRegClass);
1065  return std::make_pair(0U, &SystemZ::FP32BitRegClass);
1066 
1067  case 'v': // Vector register
1068  if (Subtarget.hasVector()) {
1069  if (VT == MVT::f32)
1070  return std::make_pair(0U, &SystemZ::VR32BitRegClass);
1071  if (VT == MVT::f64)
1072  return std::make_pair(0U, &SystemZ::VR64BitRegClass);
1073  return std::make_pair(0U, &SystemZ::VR128BitRegClass);
1074  }
1075  break;
1076  }
1077  }
1078  if (Constraint.size() > 0 && Constraint[0] == '{') {
1079  // We need to override the default register parsing for GPRs and FPRs
1080  // because the interpretation depends on VT. The internal names of
1081  // the registers are also different from the external names
1082  // (F0D and F0S instead of F0, etc.).
1083  if (Constraint[1] == 'r') {
1084  if (VT == MVT::i32)
1085  return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass,
1086  SystemZMC::GR32Regs, 16);
1087  if (VT == MVT::i128)
1088  return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass,
1089  SystemZMC::GR128Regs, 16);
1090  return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass,
1091  SystemZMC::GR64Regs, 16);
1092  }
1093  if (Constraint[1] == 'f') {
1094  if (VT == MVT::f32)
1095  return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass,
1096  SystemZMC::FP32Regs, 16);
1097  if (VT == MVT::f128)
1098  return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass,
1099  SystemZMC::FP128Regs, 16);
1100  return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass,
1101  SystemZMC::FP64Regs, 16);
1102  }
1103  if (Constraint[1] == 'v') {
1104  if (VT == MVT::f32)
1105  return parseRegisterNumber(Constraint, &SystemZ::VR32BitRegClass,
1106  SystemZMC::VR32Regs, 32);
1107  if (VT == MVT::f64)
1108  return parseRegisterNumber(Constraint, &SystemZ::VR64BitRegClass,
1109  SystemZMC::VR64Regs, 32);
1110  return parseRegisterNumber(Constraint, &SystemZ::VR128BitRegClass,
1111  SystemZMC::VR128Regs, 32);
1112  }
1113  }
1114  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1115 }
1116 
1118 LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
1119  std::vector<SDValue> &Ops,
1120  SelectionDAG &DAG) const {
1121  // Only support length 1 constraints for now.
1122  if (Constraint.length() == 1) {
1123  switch (Constraint[0]) {
1124  case 'I': // Unsigned 8-bit constant
1125  if (auto *C = dyn_cast<ConstantSDNode>(Op))
1126  if (isUInt<8>(C->getZExtValue()))
1127  Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1128  Op.getValueType()));
1129  return;
1130 
1131  case 'J': // Unsigned 12-bit constant
1132  if (auto *C = dyn_cast<ConstantSDNode>(Op))
1133  if (isUInt<12>(C->getZExtValue()))
1134  Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1135  Op.getValueType()));
1136  return;
1137 
1138  case 'K': // Signed 16-bit constant
1139  if (auto *C = dyn_cast<ConstantSDNode>(Op))
1140  if (isInt<16>(C->getSExtValue()))
1141  Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
1142  Op.getValueType()));
1143  return;
1144 
1145  case 'L': // Signed 20-bit displacement (on all targets we support)
1146  if (auto *C = dyn_cast<ConstantSDNode>(Op))
1147  if (isInt<20>(C->getSExtValue()))
1148  Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
1149  Op.getValueType()));
1150  return;
1151 
1152  case 'M': // 0x7fffffff
1153  if (auto *C = dyn_cast<ConstantSDNode>(Op))
1154  if (C->getZExtValue() == 0x7fffffff)
1155  Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1156  Op.getValueType()));
1157  return;
1158  }
1159  }
1160  TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1161 }
1162 
1163 //===----------------------------------------------------------------------===//
1164 // Calling conventions
1165 //===----------------------------------------------------------------------===//
1166 
1167 #include "SystemZGenCallingConv.inc"
1168 
1170  CallingConv::ID) const {
1171  static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1172  SystemZ::R14D, 0 };
1173  return ScratchRegs;
1174 }
1175 
1177  Type *ToType) const {
1178  return isTruncateFree(FromType, ToType);
1179 }
1180 
1182  return CI->isTailCall();
1183 }
1184 
1185 // We do not yet support 128-bit single-element vector types. If the user
1186 // attempts to use such types as function argument or return type, prefer
1187 // to error out instead of emitting code violating the ABI.
1188 static void VerifyVectorType(MVT VT, EVT ArgVT) {
1189  if (ArgVT.isVector() && !VT.isVector())
1190  report_fatal_error("Unsupported vector argument or return type");
1191 }
1192 
1194  for (unsigned i = 0; i < Ins.size(); ++i)
1195  VerifyVectorType(Ins[i].VT, Ins[i].ArgVT);
1196 }
1197 
1199  for (unsigned i = 0; i < Outs.size(); ++i)
1200  VerifyVectorType(Outs[i].VT, Outs[i].ArgVT);
1201 }
1202 
1203 // Value is a value that has been passed to us in the location described by VA
1204 // (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1205 // any loads onto Chain.
1207  CCValAssign &VA, SDValue Chain,
1208  SDValue Value) {
1209  // If the argument has been promoted from a smaller type, insert an
1210  // assertion to capture this.
1211  if (VA.getLocInfo() == CCValAssign::SExt)
1212  Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value,
1213  DAG.getValueType(VA.getValVT()));
1214  else if (VA.getLocInfo() == CCValAssign::ZExt)
1215  Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value,
1216  DAG.getValueType(VA.getValVT()));
1217 
1218  if (VA.isExtInLoc())
1219  Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value);
1220  else if (VA.getLocInfo() == CCValAssign::BCvt) {
1221  // If this is a short vector argument loaded from the stack,
1222  // extend from i64 to full vector size and then bitcast.
1223  assert(VA.getLocVT() == MVT::i64);
1224  assert(VA.getValVT().isVector());
1225  Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)});
1226  Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
1227  } else
1228  assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1229  return Value;
1230 }
1231 
1232 // Value is a value of type VA.getValVT() that we need to copy into
1233 // the location described by VA. Return a copy of Value converted to
1234 // VA.getValVT(). The caller is responsible for handling indirect values.
1236  CCValAssign &VA, SDValue Value) {
1237  switch (VA.getLocInfo()) {
1238  case CCValAssign::SExt:
1239  return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value);
1240  case CCValAssign::ZExt:
1241  return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
1242  case CCValAssign::AExt:
1243  return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
1244  case CCValAssign::BCvt:
1245  // If this is a short vector argument to be stored to the stack,
1246  // bitcast to v2i64 and then extract first element.
1247  assert(VA.getLocVT() == MVT::i64);
1248  assert(VA.getValVT().isVector());
1249  Value = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Value);
1250  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
1251  DAG.getConstant(0, DL, MVT::i32));
1252  case CCValAssign::Full:
1253  return Value;
1254  default:
1255  llvm_unreachable("Unhandled getLocInfo()");
1256  }
1257 }
1258 
1260  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1261  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1262  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1263  MachineFunction &MF = DAG.getMachineFunction();
1264  MachineFrameInfo &MFI = MF.getFrameInfo();
1266  SystemZMachineFunctionInfo *FuncInfo =
1268  auto *TFL =
1269  static_cast<const SystemZFrameLowering *>(Subtarget.getFrameLowering());
1270  EVT PtrVT = getPointerTy(DAG.getDataLayout());
1271 
1272  // Detect unsupported vector argument types.
1273  if (Subtarget.hasVector())
1274  VerifyVectorTypes(Ins);
1275 
1276  // Assign locations to all of the incoming arguments.
1278  SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1279  CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ);
1280 
1281  unsigned NumFixedGPRs = 0;
1282  unsigned NumFixedFPRs = 0;
1283  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1284  SDValue ArgValue;
1285  CCValAssign &VA = ArgLocs[I];
1286  EVT LocVT = VA.getLocVT();
1287  if (VA.isRegLoc()) {
1288  // Arguments passed in registers
1289  const TargetRegisterClass *RC;
1290  switch (LocVT.getSimpleVT().SimpleTy) {
1291  default:
1292  // Integers smaller than i64 should be promoted to i64.
1293  llvm_unreachable("Unexpected argument type");
1294  case MVT::i32:
1295  NumFixedGPRs += 1;
1296  RC = &SystemZ::GR32BitRegClass;
1297  break;
1298  case MVT::i64:
1299  NumFixedGPRs += 1;
1300  RC = &SystemZ::GR64BitRegClass;
1301  break;
1302  case MVT::f32:
1303  NumFixedFPRs += 1;
1304  RC = &SystemZ::FP32BitRegClass;
1305  break;
1306  case MVT::f64:
1307  NumFixedFPRs += 1;
1308  RC = &SystemZ::FP64BitRegClass;
1309  break;
1310  case MVT::v16i8:
1311  case MVT::v8i16:
1312  case MVT::v4i32:
1313  case MVT::v2i64:
1314  case MVT::v4f32:
1315  case MVT::v2f64:
1316  RC = &SystemZ::VR128BitRegClass;
1317  break;
1318  }
1319 
1320  unsigned VReg = MRI.createVirtualRegister(RC);
1321  MRI.addLiveIn(VA.getLocReg(), VReg);
1322  ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1323  } else {
1324  assert(VA.isMemLoc() && "Argument not register or memory");
1325 
1326  // Create the frame index object for this incoming parameter.
1327  int FI = MFI.CreateFixedObject(LocVT.getSizeInBits() / 8,
1328  VA.getLocMemOffset(), true);
1329 
1330  // Create the SelectionDAG nodes corresponding to a load
1331  // from this parameter. Unpromoted ints and floats are
1332  // passed as right-justified 8-byte values.
1333  SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1334  if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
1335  FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
1336  DAG.getIntPtrConstant(4, DL));
1337  ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
1339  }
1340 
1341  // Convert the value of the argument register into the value that's
1342  // being passed.
1343  if (VA.getLocInfo() == CCValAssign::Indirect) {
1344  InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1345  MachinePointerInfo()));
1346  // If the original argument was split (e.g. i128), we need
1347  // to load all parts of it here (using the same address).
1348  unsigned ArgIndex = Ins[I].OrigArgIndex;
1349  assert (Ins[I].PartOffset == 0);
1350  while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) {
1351  CCValAssign &PartVA = ArgLocs[I + 1];
1352  unsigned PartOffset = Ins[I + 1].PartOffset;
1353  SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1354  DAG.getIntPtrConstant(PartOffset, DL));
1355  InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1356  MachinePointerInfo()));
1357  ++I;
1358  }
1359  } else
1360  InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
1361  }
1362 
1363  if (IsVarArg) {
1364  // Save the number of non-varargs registers for later use by va_start, etc.
1365  FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
1366  FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
1367 
1368  // Likewise the address (in the form of a frame index) of where the
1369  // first stack vararg would be. The 1-byte size here is arbitrary.
1370  int64_t StackSize = CCInfo.getNextStackOffset();
1371  FuncInfo->setVarArgsFrameIndex(MFI.CreateFixedObject(1, StackSize, true));
1372 
1373  // ...and a similar frame index for the caller-allocated save area
1374  // that will be used to store the incoming registers.
1375  int64_t RegSaveOffset = TFL->getOffsetOfLocalArea();
1376  unsigned RegSaveIndex = MFI.CreateFixedObject(1, RegSaveOffset, true);
1377  FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
1378 
1379  // Store the FPR varargs in the reserved frame slots. (We store the
1380  // GPRs as part of the prologue.)
1381  if (NumFixedFPRs < SystemZ::NumArgFPRs) {
1382  SDValue MemOps[SystemZ::NumArgFPRs];
1383  for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) {
1384  unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]);
1385  int FI = MFI.CreateFixedObject(8, RegSaveOffset + Offset, true);
1386  SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1387  unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I],
1388  &SystemZ::FP64BitRegClass);
1389  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
1390  MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
1392  }
1393  // Join the stores, which are independent of one another.
1394  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
1395  makeArrayRef(&MemOps[NumFixedFPRs],
1396  SystemZ::NumArgFPRs-NumFixedFPRs));
1397  }
1398  }
1399 
1400  return Chain;
1401 }
1402 
1403 static bool canUseSiblingCall(const CCState &ArgCCInfo,
1406  // Punt if there are any indirect or stack arguments, or if the call
1407  // needs the callee-saved argument register R6, or if the call uses
1408  // the callee-saved register arguments SwiftSelf and SwiftError.
1409  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1410  CCValAssign &VA = ArgLocs[I];
1411  if (VA.getLocInfo() == CCValAssign::Indirect)
1412  return false;
1413  if (!VA.isRegLoc())
1414  return false;
1415  unsigned Reg = VA.getLocReg();
1416  if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
1417  return false;
1418  if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
1419  return false;
1420  }
1421  return true;
1422 }
1423 
1424 SDValue
1426  SmallVectorImpl<SDValue> &InVals) const {
1427  SelectionDAG &DAG = CLI.DAG;
1428  SDLoc &DL = CLI.DL;
1430  SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1432  SDValue Chain = CLI.Chain;
1433  SDValue Callee = CLI.Callee;
1434  bool &IsTailCall = CLI.IsTailCall;
1435  CallingConv::ID CallConv = CLI.CallConv;
1436  bool IsVarArg = CLI.IsVarArg;
1437  MachineFunction &MF = DAG.getMachineFunction();
1438  EVT PtrVT = getPointerTy(MF.getDataLayout());
1439 
1440  // Detect unsupported vector argument and return types.
1441  if (Subtarget.hasVector()) {
1442  VerifyVectorTypes(Outs);
1443  VerifyVectorTypes(Ins);
1444  }
1445 
1446  // Analyze the operands of the call, assigning locations to each operand.
1448  SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1449  ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
1450 
1451  // We don't support GuaranteedTailCallOpt, only automatically-detected
1452  // sibling calls.
1453  if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
1454  IsTailCall = false;
1455 
1456  // Get a count of how many bytes are to be pushed on the stack.
1457  unsigned NumBytes = ArgCCInfo.getNextStackOffset();
1458 
1459  // Mark the start of the call.
1460  if (!IsTailCall)
1461  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
1462 
1463  // Copy argument values to their designated locations.
1465  SmallVector<SDValue, 8> MemOpChains;
1466  SDValue StackPtr;
1467  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1468  CCValAssign &VA = ArgLocs[I];
1469  SDValue ArgValue = OutVals[I];
1470 
1471  if (VA.getLocInfo() == CCValAssign::Indirect) {
1472  // Store the argument in a stack slot and pass its address.
1473  SDValue SpillSlot = DAG.CreateStackTemporary(Outs[I].ArgVT);
1474  int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1475  MemOpChains.push_back(
1476  DAG.getStore(Chain, DL, ArgValue, SpillSlot,
1478  // If the original argument was split (e.g. i128), we need
1479  // to store all parts of it here (and pass just one address).
1480  unsigned ArgIndex = Outs[I].OrigArgIndex;
1481  assert (Outs[I].PartOffset == 0);
1482  while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
1483  SDValue PartValue = OutVals[I + 1];
1484  unsigned PartOffset = Outs[I + 1].PartOffset;
1485  SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
1486  DAG.getIntPtrConstant(PartOffset, DL));
1487  MemOpChains.push_back(
1488  DAG.getStore(Chain, DL, PartValue, Address,
1490  ++I;
1491  }
1492  ArgValue = SpillSlot;
1493  } else
1494  ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue);
1495 
1496  if (VA.isRegLoc())
1497  // Queue up the argument copies and emit them at the end.
1498  RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
1499  else {
1500  assert(VA.isMemLoc() && "Argument not register or memory");
1501 
1502  // Work out the address of the stack slot. Unpromoted ints and
1503  // floats are passed as right-justified 8-byte values.
1504  if (!StackPtr.getNode())
1505  StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT);
1507  if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
1508  Offset += 4;
1509  SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
1510  DAG.getIntPtrConstant(Offset, DL));
1511 
1512  // Emit the store.
1513  MemOpChains.push_back(
1514  DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
1515  }
1516  }
1517 
1518  // Join the stores, which are independent of one another.
1519  if (!MemOpChains.empty())
1520  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
1521 
1522  // Accept direct calls by converting symbolic call addresses to the
1523  // associated Target* opcodes. Force %r1 to be used for indirect
1524  // tail calls.
1525  SDValue Glue;
1526  if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1527  Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
1528  Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
1529  } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
1530  Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT);
1531  Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
1532  } else if (IsTailCall) {
1533  Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
1534  Glue = Chain.getValue(1);
1535  Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType());
1536  }
1537 
1538  // Build a sequence of copy-to-reg nodes, chained and glued together.
1539  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
1540  Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
1541  RegsToPass[I].second, Glue);
1542  Glue = Chain.getValue(1);
1543  }
1544 
1545  // The first call operand is the chain and the second is the target address.
1547  Ops.push_back(Chain);
1548  Ops.push_back(Callee);
1549 
1550  // Add argument registers to the end of the list so that they are
1551  // known live into the call.
1552  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
1553  Ops.push_back(DAG.getRegister(RegsToPass[I].first,
1554  RegsToPass[I].second.getValueType()));
1555 
1556  // Add a register mask operand representing the call-preserved registers.
1557  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
1558  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
1559  assert(Mask && "Missing call preserved mask for calling convention");
1560  Ops.push_back(DAG.getRegisterMask(Mask));
1561 
1562  // Glue the call to the argument copies, if any.
1563  if (Glue.getNode())
1564  Ops.push_back(Glue);
1565 
1566  // Emit the call.
1567  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1568  if (IsTailCall)
1569  return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
1570  Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
1571  Glue = Chain.getValue(1);
1572 
1573  // Mark the end of the call, which is glued to the call itself.
1574  Chain = DAG.getCALLSEQ_END(Chain,
1575  DAG.getConstant(NumBytes, DL, PtrVT, true),
1576  DAG.getConstant(0, DL, PtrVT, true),
1577  Glue, DL);
1578  Glue = Chain.getValue(1);
1579 
1580  // Assign locations to each value returned by this call.
1582  CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
1583  RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ);
1584 
1585  // Copy all of the result registers out of their specified physreg.
1586  for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
1587  CCValAssign &VA = RetLocs[I];
1588 
1589  // Copy the value out, gluing the copy to the end of the call sequence.
1590  SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
1591  VA.getLocVT(), Glue);
1592  Chain = RetValue.getValue(1);
1593  Glue = RetValue.getValue(2);
1594 
1595  // Convert the value of the return register into the value that's
1596  // being returned.
1597  InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue));
1598  }
1599 
1600  return Chain;
1601 }
1602 
1605  MachineFunction &MF, bool isVarArg,
1606  const SmallVectorImpl<ISD::OutputArg> &Outs,
1607  LLVMContext &Context) const {
1608  // Detect unsupported vector return types.
1609  if (Subtarget.hasVector())
1610  VerifyVectorTypes(Outs);
1611 
1612  // Special case that we cannot easily detect in RetCC_SystemZ since
1613  // i128 is not a legal type.
1614  for (auto &Out : Outs)
1615  if (Out.ArgVT == MVT::i128)
1616  return false;
1617 
1619  CCState RetCCInfo(CallConv, isVarArg, MF, RetLocs, Context);
1620  return RetCCInfo.CheckReturn(Outs, RetCC_SystemZ);
1621 }
1622 
1623 SDValue
1625  bool IsVarArg,
1626  const SmallVectorImpl<ISD::OutputArg> &Outs,
1627  const SmallVectorImpl<SDValue> &OutVals,
1628  const SDLoc &DL, SelectionDAG &DAG) const {
1629  MachineFunction &MF = DAG.getMachineFunction();
1630 
1631  // Detect unsupported vector return types.
1632  if (Subtarget.hasVector())
1633  VerifyVectorTypes(Outs);
1634 
1635  // Assign locations to each returned value.
1637  CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
1638  RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ);
1639 
1640  // Quick exit for void returns
1641  if (RetLocs.empty())
1642  return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain);
1643 
1644  // Copy the result values into the output registers.
1645  SDValue Glue;
1646  SmallVector<SDValue, 4> RetOps;
1647  RetOps.push_back(Chain);
1648  for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
1649  CCValAssign &VA = RetLocs[I];
1650  SDValue RetValue = OutVals[I];
1651 
1652  // Make the return register live on exit.
1653  assert(VA.isRegLoc() && "Can only return in registers!");
1654 
1655  // Promote the value as required.
1656  RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue);
1657 
1658  // Chain and glue the copies together.
1659  unsigned Reg = VA.getLocReg();
1660  Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue);
1661  Glue = Chain.getValue(1);
1662  RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT()));
1663  }
1664 
1665  // Update chain and glue.
1666  RetOps[0] = Chain;
1667  if (Glue.getNode())
1668  RetOps.push_back(Glue);
1669 
1670  return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps);
1671 }
1672 
1673 // Return true if Op is an intrinsic node with chain that returns the CC value
1674 // as its only (other) argument. Provide the associated SystemZISD opcode and
1675 // the mask of valid CC values if so.
1676 static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
1677  unsigned &CCValid) {
1678  unsigned Id = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
1679  switch (Id) {
1680  case Intrinsic::s390_tbegin:
1681  Opcode = SystemZISD::TBEGIN;
1682  CCValid = SystemZ::CCMASK_TBEGIN;
1683  return true;
1684 
1685  case Intrinsic::s390_tbegin_nofloat:
1686  Opcode = SystemZISD::TBEGIN_NOFLOAT;
1687  CCValid = SystemZ::CCMASK_TBEGIN;
1688  return true;
1689 
1690  case Intrinsic::s390_tend:
1691  Opcode = SystemZISD::TEND;
1692  CCValid = SystemZ::CCMASK_TEND;
1693  return true;
1694 
1695  default:
1696  return false;
1697  }
1698 }
1699 
1700 // Return true if Op is an intrinsic node without chain that returns the
1701 // CC value as its final argument. Provide the associated SystemZISD
1702 // opcode and the mask of valid CC values if so.
1703 static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
1704  unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1705  switch (Id) {
1706  case Intrinsic::s390_vpkshs:
1707  case Intrinsic::s390_vpksfs:
1708  case Intrinsic::s390_vpksgs:
1709  Opcode = SystemZISD::PACKS_CC;
1710  CCValid = SystemZ::CCMASK_VCMP;
1711  return true;
1712 
1713  case Intrinsic::s390_vpklshs:
1714  case Intrinsic::s390_vpklsfs:
1715  case Intrinsic::s390_vpklsgs:
1716  Opcode = SystemZISD::PACKLS_CC;
1717  CCValid = SystemZ::CCMASK_VCMP;
1718  return true;
1719 
1720  case Intrinsic::s390_vceqbs:
1721  case Intrinsic::s390_vceqhs:
1722  case Intrinsic::s390_vceqfs:
1723  case Intrinsic::s390_vceqgs:
1724  Opcode = SystemZISD::VICMPES;
1725  CCValid = SystemZ::CCMASK_VCMP;
1726  return true;
1727 
1728  case Intrinsic::s390_vchbs:
1729  case Intrinsic::s390_vchhs:
1730  case Intrinsic::s390_vchfs:
1731  case Intrinsic::s390_vchgs:
1732  Opcode = SystemZISD::VICMPHS;
1733  CCValid = SystemZ::CCMASK_VCMP;
1734  return true;
1735 
1736  case Intrinsic::s390_vchlbs:
1737  case Intrinsic::s390_vchlhs:
1738  case Intrinsic::s390_vchlfs:
1739  case Intrinsic::s390_vchlgs:
1740  Opcode = SystemZISD::VICMPHLS;
1741  CCValid = SystemZ::CCMASK_VCMP;
1742  return true;
1743 
1744  case Intrinsic::s390_vtm:
1745  Opcode = SystemZISD::VTM;
1746  CCValid = SystemZ::CCMASK_VCMP;
1747  return true;
1748 
1749  case Intrinsic::s390_vfaebs:
1750  case Intrinsic::s390_vfaehs:
1751  case Intrinsic::s390_vfaefs:
1752  Opcode = SystemZISD::VFAE_CC;
1753  CCValid = SystemZ::CCMASK_ANY;
1754  return true;
1755 
1756  case Intrinsic::s390_vfaezbs:
1757  case Intrinsic::s390_vfaezhs:
1758  case Intrinsic::s390_vfaezfs:
1759  Opcode = SystemZISD::VFAEZ_CC;
1760  CCValid = SystemZ::CCMASK_ANY;
1761  return true;
1762 
1763  case Intrinsic::s390_vfeebs:
1764  case Intrinsic::s390_vfeehs:
1765  case Intrinsic::s390_vfeefs:
1766  Opcode = SystemZISD::VFEE_CC;
1767  CCValid = SystemZ::CCMASK_ANY;
1768  return true;
1769 
1770  case Intrinsic::s390_vfeezbs:
1771  case Intrinsic::s390_vfeezhs:
1772  case Intrinsic::s390_vfeezfs:
1773  Opcode = SystemZISD::VFEEZ_CC;
1774  CCValid = SystemZ::CCMASK_ANY;
1775  return true;
1776 
1777  case Intrinsic::s390_vfenebs:
1778  case Intrinsic::s390_vfenehs:
1779  case Intrinsic::s390_vfenefs:
1780  Opcode = SystemZISD::VFENE_CC;
1781  CCValid = SystemZ::CCMASK_ANY;
1782  return true;
1783 
1784  case Intrinsic::s390_vfenezbs:
1785  case Intrinsic::s390_vfenezhs:
1786  case Intrinsic::s390_vfenezfs:
1787  Opcode = SystemZISD::VFENEZ_CC;
1788  CCValid = SystemZ::CCMASK_ANY;
1789  return true;
1790 
1791  case Intrinsic::s390_vistrbs:
1792  case Intrinsic::s390_vistrhs:
1793  case Intrinsic::s390_vistrfs:
1794  Opcode = SystemZISD::VISTR_CC;
1796  return true;
1797 
1798  case Intrinsic::s390_vstrcbs:
1799  case Intrinsic::s390_vstrchs:
1800  case Intrinsic::s390_vstrcfs:
1801  Opcode = SystemZISD::VSTRC_CC;
1802  CCValid = SystemZ::CCMASK_ANY;
1803  return true;
1804 
1805  case Intrinsic::s390_vstrczbs:
1806  case Intrinsic::s390_vstrczhs:
1807  case Intrinsic::s390_vstrczfs:
1808  Opcode = SystemZISD::VSTRCZ_CC;
1809  CCValid = SystemZ::CCMASK_ANY;
1810  return true;
1811 
1812  case Intrinsic::s390_vfcedbs:
1813  case Intrinsic::s390_vfcesbs:
1814  Opcode = SystemZISD::VFCMPES;
1815  CCValid = SystemZ::CCMASK_VCMP;
1816  return true;
1817 
1818  case Intrinsic::s390_vfchdbs:
1819  case Intrinsic::s390_vfchsbs:
1820  Opcode = SystemZISD::VFCMPHS;
1821  CCValid = SystemZ::CCMASK_VCMP;
1822  return true;
1823 
1824  case Intrinsic::s390_vfchedbs:
1825  case Intrinsic::s390_vfchesbs:
1826  Opcode = SystemZISD::VFCMPHES;
1827  CCValid = SystemZ::CCMASK_VCMP;
1828  return true;
1829 
1830  case Intrinsic::s390_vftcidb:
1831  case Intrinsic::s390_vftcisb:
1832  Opcode = SystemZISD::VFTCI;
1833  CCValid = SystemZ::CCMASK_VCMP;
1834  return true;
1835 
1836  case Intrinsic::s390_tdc:
1837  Opcode = SystemZISD::TDC;
1838  CCValid = SystemZ::CCMASK_TDC;
1839  return true;
1840 
1841  default:
1842  return false;
1843  }
1844 }
1845 
1846 // Emit an intrinsic with chain and an explicit CC register result.
1848  unsigned Opcode) {
1849  // Copy all operands except the intrinsic ID.
1850  unsigned NumOps = Op.getNumOperands();
1852  Ops.reserve(NumOps - 1);
1853  Ops.push_back(Op.getOperand(0));
1854  for (unsigned I = 2; I < NumOps; ++I)
1855  Ops.push_back(Op.getOperand(I));
1856 
1857  assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
1858  SDVTList RawVTs = DAG.getVTList(MVT::i32, MVT::Other);
1859  SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
1860  SDValue OldChain = SDValue(Op.getNode(), 1);
1861  SDValue NewChain = SDValue(Intr.getNode(), 1);
1862  DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain);
1863  return Intr.getNode();
1864 }
1865 
1866 // Emit an intrinsic with an explicit CC register result.
1868  unsigned Opcode) {
1869  // Copy all operands except the intrinsic ID.
1870  unsigned NumOps = Op.getNumOperands();
1872  Ops.reserve(NumOps - 1);
1873  for (unsigned I = 1; I < NumOps; ++I)
1874  Ops.push_back(Op.getOperand(I));
1875 
1876  SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops);
1877  return Intr.getNode();
1878 }
1879 
1880 // CC is a comparison that will be implemented using an integer or
1881 // floating-point comparison. Return the condition code mask for
1882 // a branch on true. In the integer case, CCMASK_CMP_UO is set for
1883 // unsigned comparisons and clear for signed ones. In the floating-point
1884 // case, CCMASK_CMP_UO has its normal mask meaning (unordered).
1885 static unsigned CCMaskForCondCode(ISD::CondCode CC) {
1886 #define CONV(X) \
1887  case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
1888  case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
1889  case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
1890 
1891  switch (CC) {
1892  default:
1893  llvm_unreachable("Invalid integer condition!");
1894 
1895  CONV(EQ);
1896  CONV(NE);
1897  CONV(GT);
1898  CONV(GE);
1899  CONV(LT);
1900  CONV(LE);
1901 
1902  case ISD::SETO: return SystemZ::CCMASK_CMP_O;
1903  case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
1904  }
1905 #undef CONV
1906 }
1907 
1908 // If C can be converted to a comparison against zero, adjust the operands
1909 // as necessary.
1910 static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
1911  if (C.ICmpType == SystemZICMP::UnsignedOnly)
1912  return;
1913 
1914  auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode());
1915  if (!ConstOp1)
1916  return;
1917 
1918  int64_t Value = ConstOp1->getSExtValue();
1919  if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
1920  (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
1921  (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
1922  (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
1923  C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
1924  C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
1925  }
1926 }
1927 
1928 // If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
1929 // adjust the operands as necessary.
1930 static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
1931  Comparison &C) {
1932  // For us to make any changes, it must a comparison between a single-use
1933  // load and a constant.
1934  if (!C.Op0.hasOneUse() ||
1935  C.Op0.getOpcode() != ISD::LOAD ||
1936  C.Op1.getOpcode() != ISD::Constant)
1937  return;
1938 
1939  // We must have an 8- or 16-bit load.
1940  auto *Load = cast<LoadSDNode>(C.Op0);
1941  unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits();
1942  if (NumBits != 8 && NumBits != 16)
1943  return;
1944 
1945  // The load must be an extending one and the constant must be within the
1946  // range of the unextended value.
1947  auto *ConstOp1 = cast<ConstantSDNode>(C.Op1);
1948  uint64_t Value = ConstOp1->getZExtValue();
1949  uint64_t Mask = (1 << NumBits) - 1;
1950  if (Load->getExtensionType() == ISD::SEXTLOAD) {
1951  // Make sure that ConstOp1 is in range of C.Op0.
1952  int64_t SignedValue = ConstOp1->getSExtValue();
1953  if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
1954  return;
1955  if (C.ICmpType != SystemZICMP::SignedOnly) {
1956  // Unsigned comparison between two sign-extended values is equivalent
1957  // to unsigned comparison between two zero-extended values.
1958  Value &= Mask;
1959  } else if (NumBits == 8) {
1960  // Try to treat the comparison as unsigned, so that we can use CLI.
1961  // Adjust CCMask and Value as necessary.
1962  if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
1963  // Test whether the high bit of the byte is set.
1964  Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
1965  else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
1966  // Test whether the high bit of the byte is clear.
1967  Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
1968  else
1969  // No instruction exists for this combination.
1970  return;
1971  C.ICmpType = SystemZICMP::UnsignedOnly;
1972  }
1973  } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
1974  if (Value > Mask)
1975  return;
1976  // If the constant is in range, we can use any comparison.
1977  C.ICmpType = SystemZICMP::Any;
1978  } else
1979  return;
1980 
1981  // Make sure that the first operand is an i32 of the right extension type.
1983  ISD::SEXTLOAD :
1984  ISD::ZEXTLOAD);
1985  if (C.Op0.getValueType() != MVT::i32 ||
1986  Load->getExtensionType() != ExtType) {
1987  C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(),
1988  Load->getBasePtr(), Load->getPointerInfo(),
1989  Load->getMemoryVT(), Load->getAlignment(),
1990  Load->getMemOperand()->getFlags());
1991  // Update the chain uses.
1992  DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), C.Op0.getValue(1));
1993  }
1994 
1995  // Make sure that the second operand is an i32 with the right value.
1996  if (C.Op1.getValueType() != MVT::i32 ||
1997  Value != ConstOp1->getZExtValue())
1998  C.Op1 = DAG.getConstant(Value, DL, MVT::i32);
1999 }
2000 
2001 // Return true if Op is either an unextended load, or a load suitable
2002 // for integer register-memory comparisons of type ICmpType.
2003 static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
2004  auto *Load = dyn_cast<LoadSDNode>(Op.getNode());
2005  if (Load) {
2006  // There are no instructions to compare a register with a memory byte.
2007  if (Load->getMemoryVT() == MVT::i8)
2008  return false;
2009  // Otherwise decide on extension type.
2010  switch (Load->getExtensionType()) {
2011  case ISD::NON_EXTLOAD:
2012  return true;
2013  case ISD::SEXTLOAD:
2014  return ICmpType != SystemZICMP::UnsignedOnly;
2015  case ISD::ZEXTLOAD:
2016  return ICmpType != SystemZICMP::SignedOnly;
2017  default:
2018  break;
2019  }
2020  }
2021  return false;
2022 }
2023 
2024 // Return true if it is better to swap the operands of C.
2025 static bool shouldSwapCmpOperands(const Comparison &C) {
2026  // Leave f128 comparisons alone, since they have no memory forms.
2027  if (C.Op0.getValueType() == MVT::f128)
2028  return false;
2029 
2030  // Always keep a floating-point constant second, since comparisons with
2031  // zero can use LOAD TEST and comparisons with other constants make a
2032  // natural memory operand.
2033  if (isa<ConstantFPSDNode>(C.Op1))
2034  return false;
2035 
2036  // Never swap comparisons with zero since there are many ways to optimize
2037  // those later.
2038  auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
2039  if (ConstOp1 && ConstOp1->getZExtValue() == 0)
2040  return false;
2041 
2042  // Also keep natural memory operands second if the loaded value is
2043  // only used here. Several comparisons have memory forms.
2044  if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse())
2045  return false;
2046 
2047  // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
2048  // In that case we generally prefer the memory to be second.
2049  if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) {
2050  // The only exceptions are when the second operand is a constant and
2051  // we can use things like CHHSI.
2052  if (!ConstOp1)
2053  return true;
2054  // The unsigned memory-immediate instructions can handle 16-bit
2055  // unsigned integers.
2056  if (C.ICmpType != SystemZICMP::SignedOnly &&
2057  isUInt<16>(ConstOp1->getZExtValue()))
2058  return false;
2059  // The signed memory-immediate instructions can handle 16-bit
2060  // signed integers.
2061  if (C.ICmpType != SystemZICMP::UnsignedOnly &&
2062  isInt<16>(ConstOp1->getSExtValue()))
2063  return false;
2064  return true;
2065  }
2066 
2067  // Try to promote the use of CGFR and CLGFR.
2068  unsigned Opcode0 = C.Op0.getOpcode();
2069  if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
2070  return true;
2071  if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
2072  return true;
2073  if (C.ICmpType != SystemZICMP::SignedOnly &&
2074  Opcode0 == ISD::AND &&
2075  C.Op0.getOperand(1).getOpcode() == ISD::Constant &&
2076  cast<ConstantSDNode>(C.Op0.getOperand(1))->getZExtValue() == 0xffffffff)
2077  return true;
2078 
2079  return false;
2080 }
2081 
2082 // Return a version of comparison CC mask CCMask in which the LT and GT
2083 // actions are swapped.
2084 static unsigned reverseCCMask(unsigned CCMask) {
2085  return ((CCMask & SystemZ::CCMASK_CMP_EQ) |
2087  (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) |
2088  (CCMask & SystemZ::CCMASK_CMP_UO));
2089 }
2090 
2091 // Check whether C tests for equality between X and Y and whether X - Y
2092 // or Y - X is also computed. In that case it's better to compare the
2093 // result of the subtraction against zero.
2094 static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
2095  Comparison &C) {
2096  if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2097  C.CCMask == SystemZ::CCMASK_CMP_NE) {
2098  for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) {
2099  SDNode *N = *I;
2100  if (N->getOpcode() == ISD::SUB &&
2101  ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) ||
2102  (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {
2103  C.Op0 = SDValue(N, 0);
2104  C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
2105  return;
2106  }
2107  }
2108  }
2109 }
2110 
2111 // Check whether C compares a floating-point value with zero and if that
2112 // floating-point value is also negated. In this case we can use the
2113 // negation to set CC, so avoiding separate LOAD AND TEST and
2114 // LOAD (NEGATIVE/COMPLEMENT) instructions.
2115 static void adjustForFNeg(Comparison &C) {
2116  auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1);
2117  if (C1 && C1->isZero()) {
2118  for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) {
2119  SDNode *N = *I;
2120  if (N->getOpcode() == ISD::FNEG) {
2121  C.Op0 = SDValue(N, 0);
2122  C.CCMask = reverseCCMask(C.CCMask);
2123  return;
2124  }
2125  }
2126  }
2127 }
2128 
2129 // Check whether C compares (shl X, 32) with 0 and whether X is
2130 // also sign-extended. In that case it is better to test the result
2131 // of the sign extension using LTGFR.
2132 //
2133 // This case is important because InstCombine transforms a comparison
2134 // with (sext (trunc X)) into a comparison with (shl X, 32).
2135 static void adjustForLTGFR(Comparison &C) {
2136  // Check for a comparison between (shl X, 32) and 0.
2137  if (C.Op0.getOpcode() == ISD::SHL &&
2138  C.Op0.getValueType() == MVT::i64 &&
2139  C.Op1.getOpcode() == ISD::Constant &&
2140  cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
2141  auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
2142  if (C1 && C1->getZExtValue() == 32) {
2143  SDValue ShlOp0 = C.Op0.getOperand(0);
2144  // See whether X has any SIGN_EXTEND_INREG uses.
2145  for (auto I = ShlOp0->use_begin(), E = ShlOp0->use_end(); I != E; ++I) {
2146  SDNode *N = *I;
2147  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
2148  cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) {
2149  C.Op0 = SDValue(N, 0);
2150  return;
2151  }
2152  }
2153  }
2154  }
2155 }
2156 
2157 // If C compares the truncation of an extending load, try to compare
2158 // the untruncated value instead. This exposes more opportunities to
2159 // reuse CC.
2160 static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
2161  Comparison &C) {
2162  if (C.Op0.getOpcode() == ISD::TRUNCATE &&
2163  C.Op0.getOperand(0).getOpcode() == ISD::LOAD &&
2164  C.Op1.getOpcode() == ISD::Constant &&
2165  cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
2166  auto *L = cast<LoadSDNode>(C.Op0.getOperand(0));
2167  if (L->getMemoryVT().getStoreSizeInBits() <= C.Op0.getValueSizeInBits()) {
2168  unsigned Type = L->getExtensionType();
2169  if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
2170  (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
2171  C.Op0 = C.Op0.getOperand(0);
2172  C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
2173  }
2174  }
2175  }
2176 }
2177 
2178 // Return true if shift operation N has an in-range constant shift value.
2179 // Store it in ShiftVal if so.
2180 static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
2181  auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1));
2182  if (!Shift)
2183  return false;
2184 
2185  uint64_t Amount = Shift->getZExtValue();
2186  if (Amount >= N.getValueSizeInBits())
2187  return false;
2188 
2189  ShiftVal = Amount;
2190  return true;
2191 }
2192 
2193 // Check whether an AND with Mask is suitable for a TEST UNDER MASK
2194 // instruction and whether the CC value is descriptive enough to handle
2195 // a comparison of type Opcode between the AND result and CmpVal.
2196 // CCMask says which comparison result is being tested and BitSize is
2197 // the number of bits in the operands. If TEST UNDER MASK can be used,
2198 // return the corresponding CC mask, otherwise return 0.
2199 static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
2200  uint64_t Mask, uint64_t CmpVal,
2201  unsigned ICmpType) {
2202  assert(Mask != 0 && "ANDs with zero should have been removed by now");
2203 
2204  // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
2205  if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) &&
2206  !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask))
2207  return 0;
2208 
2209  // Work out the masks for the lowest and highest bits.
2210  unsigned HighShift = 63 - countLeadingZeros(Mask);
2211  uint64_t High = uint64_t(1) << HighShift;
2212  uint64_t Low = uint64_t(1) << countTrailingZeros(Mask);
2213 
2214  // Signed ordered comparisons are effectively unsigned if the sign
2215  // bit is dropped.
2216  bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
2217 
2218  // Check for equality comparisons with 0, or the equivalent.
2219  if (CmpVal == 0) {
2220  if (CCMask == SystemZ::CCMASK_CMP_EQ)
2221  return SystemZ::CCMASK_TM_ALL_0;
2222  if (CCMask == SystemZ::CCMASK_CMP_NE)
2224  }
2225  if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
2226  if (CCMask == SystemZ::CCMASK_CMP_LT)
2227  return SystemZ::CCMASK_TM_ALL_0;
2228  if (CCMask == SystemZ::CCMASK_CMP_GE)
2230  }
2231  if (EffectivelyUnsigned && CmpVal < Low) {
2232  if (CCMask == SystemZ::CCMASK_CMP_LE)
2233  return SystemZ::CCMASK_TM_ALL_0;
2234  if (CCMask == SystemZ::CCMASK_CMP_GT)
2236  }
2237 
2238  // Check for equality comparisons with the mask, or the equivalent.
2239  if (CmpVal == Mask) {
2240  if (CCMask == SystemZ::CCMASK_CMP_EQ)
2241  return SystemZ::CCMASK_TM_ALL_1;
2242  if (CCMask == SystemZ::CCMASK_CMP_NE)
2244  }
2245  if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
2246  if (CCMask == SystemZ::CCMASK_CMP_GT)
2247  return SystemZ::CCMASK_TM_ALL_1;
2248  if (CCMask == SystemZ::CCMASK_CMP_LE)
2250  }
2251  if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
2252  if (CCMask == SystemZ::CCMASK_CMP_GE)
2253  return SystemZ::CCMASK_TM_ALL_1;
2254  if (CCMask == SystemZ::CCMASK_CMP_LT)
2256  }
2257 
2258  // Check for ordered comparisons with the top bit.
2259  if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
2260  if (CCMask == SystemZ::CCMASK_CMP_LE)
2261  return SystemZ::CCMASK_TM_MSB_0;
2262  if (CCMask == SystemZ::CCMASK_CMP_GT)
2263  return SystemZ::CCMASK_TM_MSB_1;
2264  }
2265  if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
2266  if (CCMask == SystemZ::CCMASK_CMP_LT)
2267  return SystemZ::CCMASK_TM_MSB_0;
2268  if (CCMask == SystemZ::CCMASK_CMP_GE)
2269  return SystemZ::CCMASK_TM_MSB_1;
2270  }
2271 
2272  // If there are just two bits, we can do equality checks for Low and High
2273  // as well.
2274  if (Mask == Low + High) {
2275  if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
2277  if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
2279  if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
2281  if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
2283  }
2284 
2285  // Looks like we've exhausted our options.
2286  return 0;
2287 }
2288 
2289 // See whether C can be implemented as a TEST UNDER MASK instruction.
2290 // Update the arguments with the TM version if so.
2291 static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
2292  Comparison &C) {
2293  // Check that we have a comparison with a constant.
2294  auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
2295  if (!ConstOp1)
2296  return;
2297  uint64_t CmpVal = ConstOp1->getZExtValue();
2298 
2299  // Check whether the nonconstant input is an AND with a constant mask.
2300  Comparison NewC(C);
2301  uint64_t MaskVal;
2302  ConstantSDNode *Mask = nullptr;
2303  if (C.Op0.getOpcode() == ISD::AND) {
2304  NewC.Op0 = C.Op0.getOperand(0);
2305  NewC.Op1 = C.Op0.getOperand(1);
2306  Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
2307  if (!Mask)
2308  return;
2309  MaskVal = Mask->getZExtValue();
2310  } else {
2311  // There is no instruction to compare with a 64-bit immediate
2312  // so use TMHH instead if possible. We need an unsigned ordered
2313  // comparison with an i64 immediate.
2314  if (NewC.Op0.getValueType() != MVT::i64 ||
2315  NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
2316  NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
2317  NewC.ICmpType == SystemZICMP::SignedOnly)
2318  return;
2319  // Convert LE and GT comparisons into LT and GE.
2320  if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
2321  NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
2322  if (CmpVal == uint64_t(-1))
2323  return;
2324  CmpVal += 1;
2325  NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2326  }
2327  // If the low N bits of Op1 are zero than the low N bits of Op0 can
2328  // be masked off without changing the result.
2329  MaskVal = -(CmpVal & -CmpVal);
2330  NewC.ICmpType = SystemZICMP::UnsignedOnly;
2331  }
2332  if (!MaskVal)
2333  return;
2334 
2335  // Check whether the combination of mask, comparison value and comparison
2336  // type are suitable.
2337  unsigned BitSize = NewC.Op0.getValueSizeInBits();
2338  unsigned NewCCMask, ShiftVal;
2339  if (NewC.ICmpType != SystemZICMP::SignedOnly &&
2340  NewC.Op0.getOpcode() == ISD::SHL &&
2341  isSimpleShift(NewC.Op0, ShiftVal) &&
2342  (MaskVal >> ShiftVal != 0) &&
2343  ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
2344  (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
2345  MaskVal >> ShiftVal,
2346  CmpVal >> ShiftVal,
2347  SystemZICMP::Any))) {
2348  NewC.Op0 = NewC.Op0.getOperand(0);
2349  MaskVal >>= ShiftVal;
2350  } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
2351  NewC.Op0.getOpcode() == ISD::SRL &&
2352  isSimpleShift(NewC.Op0, ShiftVal) &&
2353  (MaskVal << ShiftVal != 0) &&
2354  ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
2355  (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
2356  MaskVal << ShiftVal,
2357  CmpVal << ShiftVal,
2359  NewC.Op0 = NewC.Op0.getOperand(0);
2360  MaskVal <<= ShiftVal;
2361  } else {
2362  NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal,
2363  NewC.ICmpType);
2364  if (!NewCCMask)
2365  return;
2366  }
2367 
2368  // Go ahead and make the change.
2369  C.Opcode = SystemZISD::TM;
2370  C.Op0 = NewC.Op0;
2371  if (Mask && Mask->getZExtValue() == MaskVal)
2372  C.Op1 = SDValue(Mask, 0);
2373  else
2374  C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
2375  C.CCValid = SystemZ::CCMASK_TM;
2376  C.CCMask = NewCCMask;
2377 }
2378 
2379 // See whether the comparison argument contains a redundant AND
2380 // and remove it if so. This sometimes happens due to the generic
2381 // BRCOND expansion.
2382 static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
2383  Comparison &C) {
2384  if (C.Op0.getOpcode() != ISD::AND)
2385  return;
2386  auto *Mask = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
2387  if (!Mask)
2388  return;
2389  KnownBits Known = DAG.computeKnownBits(C.Op0.getOperand(0));
2390  if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
2391  return;
2392 
2393  C.Op0 = C.Op0.getOperand(0);
2394 }
2395 
2396 // Return a Comparison that tests the condition-code result of intrinsic
2397 // node Call against constant integer CC using comparison code Cond.
2398 // Opcode is the opcode of the SystemZISD operation for the intrinsic
2399 // and CCValid is the set of possible condition-code results.
2400 static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
2401  SDValue Call, unsigned CCValid, uint64_t CC,
2402  ISD::CondCode Cond) {
2403  Comparison C(Call, SDValue());
2404  C.Opcode = Opcode;
2405  C.CCValid = CCValid;
2406  if (Cond == ISD::SETEQ)
2407  // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
2408  C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
2409  else if (Cond == ISD::SETNE)
2410  // ...and the inverse of that.
2411  C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
2412  else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
2413  // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
2414  // always true for CC>3.
2415  C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
2416  else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
2417  // ...and the inverse of that.
2418  C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
2419  else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
2420  // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
2421  // always true for CC>3.
2422  C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
2423  else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
2424  // ...and the inverse of that.
2425  C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
2426  else
2427  llvm_unreachable("Unexpected integer comparison type");
2428  C.CCMask &= CCValid;
2429  return C;
2430 }
2431 
2432 // Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
2433 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
2434  ISD::CondCode Cond, const SDLoc &DL) {
2435  if (CmpOp1.getOpcode() == ISD::Constant) {
2436  uint64_t Constant = cast<ConstantSDNode>(CmpOp1)->getZExtValue();
2437  unsigned Opcode, CCValid;
2438  if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
2439  CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) &&
2440  isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid))
2441  return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
2442  if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
2443  CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
2444  isIntrinsicWithCC(CmpOp0, Opcode, CCValid))
2445  return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
2446  }
2447  Comparison C(CmpOp0, CmpOp1);
2448  C.CCMask = CCMaskForCondCode(Cond);
2449  if (C.Op0.getValueType().isFloatingPoint()) {
2450  C.CCValid = SystemZ::CCMASK_FCMP;
2451  C.Opcode = SystemZISD::FCMP;
2452  adjustForFNeg(C);
2453  } else {
2454  C.CCValid = SystemZ::CCMASK_ICMP;
2455  C.Opcode = SystemZISD::ICMP;
2456  // Choose the type of comparison. Equality and inequality tests can
2457  // use either signed or unsigned comparisons. The choice also doesn't
2458  // matter if both sign bits are known to be clear. In those cases we
2459  // want to give the main isel code the freedom to choose whichever
2460  // form fits best.
2461  if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2462  C.CCMask == SystemZ::CCMASK_CMP_NE ||
2463  (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1)))
2464  C.ICmpType = SystemZICMP::Any;
2465  else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
2466  C.ICmpType = SystemZICMP::UnsignedOnly;
2467  else
2468  C.ICmpType = SystemZICMP::SignedOnly;
2469  C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
2470  adjustForRedundantAnd(DAG, DL, C);
2471  adjustZeroCmp(DAG, DL, C);
2472  adjustSubwordCmp(DAG, DL, C);
2473  adjustForSubtraction(DAG, DL, C);
2474  adjustForLTGFR(C);
2475  adjustICmpTruncate(DAG, DL, C);
2476  }
2477 
2478  if (shouldSwapCmpOperands(C)) {
2479  std::swap(C.Op0, C.Op1);
2480  C.CCMask = reverseCCMask(C.CCMask);
2481  }
2482 
2483  adjustForTestUnderMask(DAG, DL, C);
2484  return C;
2485 }
2486 
2487 // Emit the comparison instruction described by C.
2488 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2489  if (!C.Op1.getNode()) {
2490  SDNode *Node;
2491  switch (C.Op0.getOpcode()) {
2493  Node = emitIntrinsicWithCCAndChain(DAG, C.Op0, C.Opcode);
2494  return SDValue(Node, 0);
2496  Node = emitIntrinsicWithCC(DAG, C.Op0, C.Opcode);
2497  return SDValue(Node, Node->getNumValues() - 1);
2498  default:
2499  llvm_unreachable("Invalid comparison operands");
2500  }
2501  }
2502  if (C.Opcode == SystemZISD::ICMP)
2503  return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1,
2504  DAG.getConstant(C.ICmpType, DL, MVT::i32));
2505  if (C.Opcode == SystemZISD::TM) {
2506  bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
2507  bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
2508  return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
2509  DAG.getConstant(RegisterOnly, DL, MVT::i32));
2510  }
2511  return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
2512 }
2513 
2514 // Implement a 32-bit *MUL_LOHI operation by extending both operands to
2515 // 64 bits. Extend is the extension type to use. Store the high part
2516 // in Hi and the low part in Lo.
2517 static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
2518  SDValue Op0, SDValue Op1, SDValue &Hi,
2519  SDValue &Lo) {
2520  Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
2521  Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
2522  SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
2523  Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2524  DAG.getConstant(32, DL, MVT::i64));
2525  Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
2526  Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
2527 }
2528 
2529 // Lower a binary operation that produces two VT results, one in each
2530 // half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
2531 // and Opcode performs the GR128 operation. Store the even register result
2532 // in Even and the odd register result in Odd.
2533 static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
2534  unsigned Opcode, SDValue Op0, SDValue Op1,
2535  SDValue &Even, SDValue &Odd) {
2536  SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1);
2537  bool Is32Bit = is32Bit(VT);
2538  Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
2539  Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
2540 }
2541 
2542 // Return an i32 value that is 1 if the CC value produced by CCReg is
2543 // in the mask CCMask and 0 otherwise. CC is known to have a value
2544 // in CCValid, so other values can be ignored.
2545 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
2546  unsigned CCValid, unsigned CCMask) {
2547  SDValue Ops[] = { DAG.getConstant(1, DL, MVT::i32),
2548  DAG.getConstant(0, DL, MVT::i32),
2549  DAG.getConstant(CCValid, DL, MVT::i32),
2550  DAG.getConstant(CCMask, DL, MVT::i32), CCReg };
2551  return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
2552 }
2553 
2554 // Return the SystemISD vector comparison operation for CC, or 0 if it cannot
2555 // be done directly. IsFP is true if CC is for a floating-point rather than
2556 // integer comparison.
2557 static unsigned getVectorComparison(ISD::CondCode CC, bool IsFP) {
2558  switch (CC) {
2559  case ISD::SETOEQ:
2560  case ISD::SETEQ:
2561  return IsFP ? SystemZISD::VFCMPE : SystemZISD::VICMPE;
2562 
2563  case ISD::SETOGE:
2564  case ISD::SETGE:
2565  return IsFP ? SystemZISD::VFCMPHE : static_cast<SystemZISD::NodeType>(0);
2566 
2567  case ISD::SETOGT:
2568  case ISD::SETGT:
2569  return IsFP ? SystemZISD::VFCMPH : SystemZISD::VICMPH;
2570 
2571  case ISD::SETUGT:
2572  return IsFP ? static_cast<SystemZISD::NodeType>(0) : SystemZISD::VICMPHL;
2573 
2574  default:
2575  return 0;
2576  }
2577 }
2578 
2579 // Return the SystemZISD vector comparison operation for CC or its inverse,
2580 // or 0 if neither can be done directly. Indicate in Invert whether the
2581 // result is for the inverse of CC. IsFP is true if CC is for a
2582 // floating-point rather than integer comparison.
2583 static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, bool IsFP,
2584  bool &Invert) {
2585  if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
2586  Invert = false;
2587  return Opcode;
2588  }
2589 
2590  CC = ISD::getSetCCInverse(CC, !IsFP);
2591  if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
2592  Invert = true;
2593  return Opcode;
2594  }
2595 
2596  return 0;
2597 }
2598 
2599 // Return a v2f64 that contains the extended form of elements Start and Start+1
2600 // of v4f32 value Op.
2601 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
2602  SDValue Op) {
2603  int Mask[] = { Start, -1, Start + 1, -1 };
2604  Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
2605  return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
2606 }
2607 
2608 // Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
2609 // producing a result of type VT.
2610 SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
2611  const SDLoc &DL, EVT VT,
2612  SDValue CmpOp0,
2613  SDValue CmpOp1) const {
2614  // There is no hardware support for v4f32 (unless we have the vector
2615  // enhancements facility 1), so extend the vector into two v2f64s
2616  // and compare those.
2617  if (CmpOp0.getValueType() == MVT::v4f32 &&
2618  !Subtarget.hasVectorEnhancements1()) {
2619  SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0);
2620  SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0);
2621  SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1);
2622  SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1);
2623  SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
2624  SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
2625  return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
2626  }
2627  return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
2628 }
2629 
2630 // Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
2631 // an integer mask of type VT.
2632 SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
2633  const SDLoc &DL, EVT VT,
2634  ISD::CondCode CC,
2635  SDValue CmpOp0,
2636  SDValue CmpOp1) const {
2637  bool IsFP = CmpOp0.getValueType().isFloatingPoint();
2638  bool Invert = false;
2639  SDValue Cmp;
2640  switch (CC) {
2641  // Handle tests for order using (or (ogt y x) (oge x y)).
2642  case ISD::SETUO:
2643  Invert = true;
2645  case ISD::SETO: {
2646  assert(IsFP && "Unexpected integer comparison");
2647  SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
2648  SDValue GE = getVectorCmp(DAG, SystemZISD::VFCMPHE, DL, VT, CmpOp0, CmpOp1);
2649  Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
2650  break;
2651  }
2652 
2653  // Handle <> tests using (or (ogt y x) (ogt x y)).
2654  case ISD::SETUEQ:
2655  Invert = true;
2657  case ISD::SETONE: {
2658  assert(IsFP && "Unexpected integer comparison");
2659  SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
2660  SDValue GT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp0, CmpOp1);
2661  Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
2662  break;
2663  }
2664 
2665  // Otherwise a single comparison is enough. It doesn't really
2666  // matter whether we try the inversion or the swap first, since
2667  // there are no cases where both work.
2668  default:
2669  if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
2670  Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1);
2671  else {
2673  if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
2674  Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0);
2675  else
2676  llvm_unreachable("Unhandled comparison");
2677  }
2678  break;
2679  }
2680  if (Invert) {
2681  SDValue Mask =
2682  DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64));
2683  Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
2684  }
2685  return Cmp;
2686 }
2687 
2688 SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
2689  SelectionDAG &DAG) const {
2690  SDValue CmpOp0 = Op.getOperand(0);
2691  SDValue CmpOp1 = Op.getOperand(1);
2692  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
2693  SDLoc DL(Op);
2694  EVT VT = Op.getValueType();
2695  if (VT.isVector())
2696  return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
2697 
2698  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2699  SDValue CCReg = emitCmp(DAG, DL, C);
2700  return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
2701 }
2702 
2703 SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
2704  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
2705  SDValue CmpOp0 = Op.getOperand(2);
2706  SDValue CmpOp1 = Op.getOperand(3);
2707  SDValue Dest = Op.getOperand(4);
2708  SDLoc DL(Op);
2709 
2710  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2711  SDValue CCReg = emitCmp(DAG, DL, C);
2712  return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(),
2713  Op.getOperand(0), DAG.getConstant(C.CCValid, DL, MVT::i32),
2714  DAG.getConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
2715 }
2716 
2717 // Return true if Pos is CmpOp and Neg is the negative of CmpOp,
2718 // allowing Pos and Neg to be wider than CmpOp.
2719 static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
2720  return (Neg.getOpcode() == ISD::SUB &&
2721  Neg.getOperand(0).getOpcode() == ISD::Constant &&
2722  cast<ConstantSDNode>(Neg.getOperand(0))->getZExtValue() == 0 &&
2723  Neg.getOperand(1) == Pos &&
2724  (Pos == CmpOp ||
2725  (Pos.getOpcode() == ISD::SIGN_EXTEND &&
2726  Pos.getOperand(0) == CmpOp)));
2727 }
2728 
2729 // Return the absolute or negative absolute of Op; IsNegative decides which.
2730 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
2731  bool IsNegative) {
2732  Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op);
2733  if (IsNegative)
2734  Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
2735  DAG.getConstant(0, DL, Op.getValueType()), Op);
2736  return Op;
2737 }
2738 
2739 SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
2740  SelectionDAG &DAG) const {
2741  SDValue CmpOp0 = Op.getOperand(0);
2742  SDValue CmpOp1 = Op.getOperand(1);
2743  SDValue TrueOp = Op.getOperand(2);
2744  SDValue FalseOp = Op.getOperand(3);
2745  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
2746  SDLoc DL(Op);
2747 
2748  Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2749 
2750  // Check for absolute and negative-absolute selections, including those
2751  // where the comparison value is sign-extended (for LPGFR and LNGFR).
2752  // This check supplements the one in DAGCombiner.
2753  if (C.Opcode == SystemZISD::ICMP &&
2754  C.CCMask != SystemZ::CCMASK_CMP_EQ &&
2755  C.CCMask != SystemZ::CCMASK_CMP_NE &&
2756  C.Op1.getOpcode() == ISD::Constant &&
2757  cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
2758  if (isAbsolute(C.Op0, TrueOp, FalseOp))
2759  return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
2760  if (isAbsolute(C.Op0, FalseOp, TrueOp))
2761  return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
2762  }
2763 
2764  SDValue CCReg = emitCmp(DAG, DL, C);
2765  SDValue Ops[] = {TrueOp, FalseOp, DAG.getConstant(C.CCValid, DL, MVT::i32),
2766  DAG.getConstant(C.CCMask, DL, MVT::i32), CCReg};
2767 
2768  return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
2769 }
2770 
2771 SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
2772  SelectionDAG &DAG) const {
2773  SDLoc DL(Node);
2774  const GlobalValue *GV = Node->getGlobal();
2775  int64_t Offset = Node->getOffset();
2776  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2778 
2779  SDValue Result;
2780  if (Subtarget.isPC32DBLSymbol(GV, CM)) {
2781  // Assign anchors at 1<<12 byte boundaries.
2782  uint64_t Anchor = Offset & ~uint64_t(0xfff);
2783  Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor);
2784  Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2785 
2786  // The offset can be folded into the address if it is aligned to a halfword.
2787  Offset -= Anchor;
2788  if (Offset != 0 && (Offset & 1) == 0) {
2789  SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
2790  Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result);
2791  Offset = 0;
2792  }
2793  } else {
2794  Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT);
2795  Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2796  Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
2798  }
2799 
2800  // If there was a non-zero offset that we didn't fold, create an explicit
2801  // addition for it.
2802  if (Offset != 0)
2803  Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
2804  DAG.getConstant(Offset, DL, PtrVT));
2805 
2806  return Result;
2807 }
2808 
2809 SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
2810  SelectionDAG &DAG,
2811  unsigned Opcode,
2812  SDValue GOTOffset) const {
2813  SDLoc DL(Node);
2814  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2815  SDValue Chain = DAG.getEntryNode();
2816  SDValue Glue;
2817 
2818  // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
2819  SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
2820  Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
2821  Glue = Chain.getValue(1);
2822  Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
2823  Glue = Chain.getValue(1);
2824 
2825  // The first call operand is the chain and the second is the TLS symbol.
2827  Ops.push_back(Chain);
2828  Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,
2829  Node->getValueType(0),
2830  0, 0));
2831 
2832  // Add argument registers to the end of the list so that they are
2833  // known live into the call.
2834  Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT));
2835  Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT));
2836 
2837  // Add a register mask operand representing the call-preserved registers.
2838  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2839  const uint32_t *Mask =
2841  assert(Mask && "Missing call preserved mask for calling convention");
2842  Ops.push_back(DAG.getRegisterMask(Mask));
2843 
2844  // Glue the call to the argument copies.
2845  Ops.push_back(Glue);
2846 
2847  // Emit the call.
2848  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2849  Chain = DAG.getNode(Opcode, DL, NodeTys, Ops);
2850  Glue = Chain.getValue(1);
2851 
2852  // Copy the return value from %r2.
2853  return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
2854 }
2855 
2856 SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
2857  SelectionDAG &DAG) const {
2858  SDValue Chain = DAG.getEntryNode();
2859  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2860 
2861  // The high part of the thread pointer is in access register 0.
2862  SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
2863  TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
2864 
2865  // The low part of the thread pointer is in access register 1.
2866  SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
2867  TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
2868 
2869  // Merge them into a single 64-bit address.
2870  SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
2871  DAG.getConstant(32, DL, PtrVT));
2872  return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
2873 }
2874 
2875 SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
2876  SelectionDAG &DAG) const {
2877  if (DAG.getTarget().useEmulatedTLS())
2878  return LowerToTLSEmulatedModel(Node, DAG);
2879  SDLoc DL(Node);
2880  const GlobalValue *GV = Node->getGlobal();
2881  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2882  TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
2883 
2884  SDValue TP = lowerThreadPointer(DL, DAG);
2885 
2886  // Get the offset of GA from the thread pointer, based on the TLS model.
2887  SDValue Offset;
2888  switch (model) {
2889  case TLSModel::GeneralDynamic: {
2890  // Load the GOT offset of the tls_index (module ID / per-symbol offset).
2893 
2894  Offset = DAG.getConstantPool(CPV, PtrVT, 8);
2895  Offset = DAG.getLoad(
2896  PtrVT, DL, DAG.getEntryNode(), Offset,
2898 
2899  // Call __tls_get_offset to retrieve the offset.
2900  Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset);
2901  break;
2902  }
2903 
2904  case TLSModel::LocalDynamic: {
2905  // Load the GOT offset of the module ID.
2908 
2909  Offset = DAG.getConstantPool(CPV, PtrVT, 8);
2910  Offset = DAG.getLoad(
2911  PtrVT, DL, DAG.getEntryNode(), Offset,
2913 
2914  // Call __tls_get_offset to retrieve the module base offset.
2915  Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset);
2916 
2917  // Note: The SystemZLDCleanupPass will remove redundant computations
2918  // of the module base offset. Count total number of local-dynamic
2919  // accesses to trigger execution of that pass.
2923 
2924  // Add the per-symbol offset.
2926 
2927  SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, 8);
2928  DTPOffset = DAG.getLoad(
2929  PtrVT, DL, DAG.getEntryNode(), DTPOffset,
2931 
2932  Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset);
2933  break;
2934  }
2935 
2936  case TLSModel::InitialExec: {
2937  // Load the offset from the GOT.
2938  Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2940  Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset);
2941  Offset =
2942  DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset,
2944  break;
2945  }
2946 
2947  case TLSModel::LocalExec: {
2948  // Force the offset into the constant pool and load it from there.
2951 
2952  Offset = DAG.getConstantPool(CPV, PtrVT, 8);
2953  Offset = DAG.getLoad(
2954  PtrVT, DL, DAG.getEntryNode(), Offset,
2956  break;
2957  }
2958  }
2959 
2960  // Add the base and offset together.
2961  return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset);
2962 }
2963 
2964 SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
2965  SelectionDAG &DAG) const {
2966  SDLoc DL(Node);
2967  const BlockAddress *BA = Node->getBlockAddress();
2968  int64_t Offset = Node->getOffset();
2969  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2970 
2971  SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
2972  Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2973  return Result;
2974 }
2975 
2976 SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
2977  SelectionDAG &DAG) const {
2978  SDLoc DL(JT);
2979  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2980  SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
2981 
2982  // Use LARL to load the address of the table.
2983  return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2984 }
2985 
2986 SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
2987  SelectionDAG &DAG) const {
2988  SDLoc DL(CP);
2989  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2990 
2991  SDValue Result;
2992  if (CP->isMachineConstantPoolEntry())
2993  Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT,
2994  CP->getAlignment());
2995  else
2996  Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT,
2997  CP->getAlignment(), CP->getOffset());
2998 
2999  // Use LARL to load the address of the constant pool entry.
3000  return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3001 }
3002 
3003 SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
3004  SelectionDAG &DAG) const {
3005  MachineFunction &MF = DAG.getMachineFunction();
3006  MachineFrameInfo &MFI = MF.getFrameInfo();
3007  MFI.setFrameAddressIsTaken(true);
3008 
3009  SDLoc DL(Op);
3010  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3011  EVT PtrVT = getPointerTy(DAG.getDataLayout());
3012 
3013  // If the back chain frame index has not been allocated yet, do so.
3015  int BackChainIdx = FI->getFramePointerSaveIndex();
3016  if (!BackChainIdx) {
3017  // By definition, the frame address is the address of the back chain.
3018  BackChainIdx = MFI.CreateFixedObject(8, -SystemZMC::CallFrameSize, false);
3019  FI->setFramePointerSaveIndex(BackChainIdx);
3020  }
3021  SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
3022 
3023  // FIXME The frontend should detect this case.
3024  if (Depth > 0) {
3025  report_fatal_error("Unsupported stack frame traversal count");
3026  }
3027 
3028  return BackChain;
3029 }
3030 
3031 SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
3032  SelectionDAG &DAG) const {
3033  MachineFunction &MF = DAG.getMachineFunction();
3034  MachineFrameInfo &MFI = MF.getFrameInfo();
3035  MFI.setReturnAddressIsTaken(true);
3036 
3038  return SDValue();
3039 
3040  SDLoc DL(Op);
3041  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3042  EVT PtrVT = getPointerTy(DAG.getDataLayout());
3043 
3044  // FIXME The frontend should detect this case.
3045  if (Depth > 0) {
3046  report_fatal_error("Unsupported stack frame traversal count");
3047  }
3048 
3049  // Return R14D, which has the return address. Mark it an implicit live-in.
3050  unsigned LinkReg = MF.addLiveIn(SystemZ::R14D, &SystemZ::GR64BitRegClass);
3051  return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT);
3052 }
3053 
3054 SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
3055  SelectionDAG &DAG) const {
3056  SDLoc DL(Op);
3057  SDValue In = Op.getOperand(0);
3058  EVT InVT = In.getValueType();
3059  EVT ResVT = Op.getValueType();
3060 
3061  // Convert loads directly. This is normally done by DAGCombiner,
3062  // but we need this case for bitcasts that are created during lowering
3063  // and which are then lowered themselves.
3064  if (auto *LoadN = dyn_cast<LoadSDNode>(In))
3065  if (ISD::isNormalLoad(LoadN)) {
3066  SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
3067  LoadN->getBasePtr(), LoadN->getMemOperand());
3068  // Update the chain uses.
3069  DAG.ReplaceAllUsesOfValueWith(SDValue(LoadN, 1), NewLoad.getValue(1));
3070  return NewLoad;
3071  }
3072 
3073  if (InVT == MVT::i32 && ResVT == MVT::f32) {
3074  SDValue In64;
3075  if (Subtarget.hasHighWord()) {
3076  SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL,
3077  MVT::i64);
3078  In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
3079  MVT::i64, SDValue(U64, 0), In);
3080  } else {
3081  In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
3082  In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
3083  DAG.getConstant(32, DL, MVT::i64));
3084  }
3085  SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
3086  return DAG.getTargetExtractSubreg(SystemZ::subreg_h32,
3087  DL, MVT::f32, Out64);
3088  }
3089  if (InVT == MVT::f32 && ResVT == MVT::i32) {
3090  SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
3091  SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
3092  MVT::f64, SDValue(U64, 0), In);
3093  SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
3094  if (Subtarget.hasHighWord())
3095  return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
3096  MVT::i32, Out64);
3097  SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
3098  DAG.getConstant(32, DL, MVT::i64));
3099  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
3100  }
3101  llvm_unreachable("Unexpected bitcast combination");
3102 }
3103 
3104 SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
3105  SelectionDAG &DAG) const {
3106  MachineFunction &MF = DAG.getMachineFunction();
3107  SystemZMachineFunctionInfo *FuncInfo =
3109  EVT PtrVT = getPointerTy(DAG.getDataLayout());
3110 
3111  SDValue Chain = Op.getOperand(0);
3112  SDValue Addr = Op.getOperand(1);
3113  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3114  SDLoc DL(Op);
3115 
3116  // The initial values of each field.
3117  const unsigned NumFields = 4;
3118  SDValue Fields[NumFields] = {
3119  DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
3120  DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
3121  DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),
3122  DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)
3123  };
3124 
3125  // Store each field into its respective slot.
3126  SDValue MemOps[NumFields];
3127  unsigned Offset = 0;
3128  for (unsigned I = 0; I < NumFields; ++I) {
3129  SDValue FieldAddr = Addr;
3130  if (Offset != 0)
3131  FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
3132  DAG.getIntPtrConstant(Offset, DL));
3133  MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
3134  MachinePointerInfo(SV, Offset));
3135  Offset += 8;
3136  }
3137  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3138 }
3139 
3140 SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
3141  SelectionDAG &DAG) const {
3142  SDValue Chain = Op.getOperand(0);
3143  SDValue DstPtr = Op.getOperand(1);
3144  SDValue SrcPtr = Op.getOperand(2);
3145  const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
3146  const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
3147  SDLoc DL(Op);
3148 
3149  return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32, DL),
3150  /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false,
3151  /*isTailCall*/false,
3152  MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
3153 }
3154 
3155 SDValue SystemZTargetLowering::
3156 lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
3157  const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
3158  MachineFunction &MF = DAG.getMachineFunction();
3159  bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");
3160  bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
3161 
3162  SDValue Chain = Op.getOperand(0);
3163  SDValue Size = Op.getOperand(1);
3164  SDValue Align = Op.getOperand(2);
3165  SDLoc DL(Op);
3166 
3167  // If user has set the no alignment function attribute, ignore
3168  // alloca alignments.
3169  uint64_t AlignVal = (RealignOpt ?
3170  dyn_cast<ConstantSDNode>(Align)->getZExtValue() : 0);
3171 
3172  uint64_t StackAlign = TFI->getStackAlignment();
3173  uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
3174  uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
3175 
3176  unsigned SPReg = getStackPointerRegisterToSaveRestore();
3177  SDValue NeededSpace = Size;
3178 
3179  // Get a reference to the stack pointer.
3180  SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
3181 
3182  // If we need a backchain, save it now.
3183  SDValue Backchain;
3184  if (StoreBackchain)
3185  Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo());
3186 
3187  // Add extra space for alignment if needed.
3188  if (ExtraAlignSpace)
3189  NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace,
3190  DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
3191 
3192  // Get the new stack pointer value.
3193  SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
3194 
3195  // Copy the new stack pointer back.
3196  Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP);
3197 
3198  // The allocated data lives above the 160 bytes allocated for the standard
3199  // frame, plus any outgoing stack arguments. We don't know how much that
3200  // amounts to yet, so emit a special ADJDYNALLOC placeholder.
3201  SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
3202  SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
3203 
3204  // Dynamically realign if needed.
3205  if (RequiredAlign > StackAlign) {
3206  Result =
3207  DAG.getNode(ISD::ADD, DL, MVT::i64, Result,
3208  DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
3209  Result =
3210  DAG.getNode(ISD::AND, DL, MVT::i64, Result,
3211  DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));
3212  }
3213 
3214  if (StoreBackchain)
3215  Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo());
3216 
3217  SDValue Ops[2] = { Result, Chain };
3218  return DAG.getMergeValues(Ops, DL);
3219 }
3220 
3221 SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
3222  SDValue Op, SelectionDAG &DAG) const {
3223  SDLoc DL(Op);
3224 
3225  return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
3226 }
3227 
3228 SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
3229  SelectionDAG &DAG) const {
3230  EVT VT = Op.getValueType();
3231  SDLoc DL(Op);
3232  SDValue Ops[2];
3233  if (is32Bit(VT))
3234  // Just do a normal 64-bit multiplication and extract the results.
3235  // We define this so that it can be used for constant division.
3237  Op.getOperand(1), Ops[1], Ops[0]);
3238  else if (Subtarget.hasMiscellaneousExtensions2())
3239  // SystemZISD::SMUL_LOHI returns the low result in the odd register and
3240  // the high result in the even register. ISD::SMUL_LOHI is defined to
3241  // return the low half first, so the results are in reverse order.
3243  Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
3244  else {
3245  // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
3246  //
3247  // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
3248  //
3249  // but using the fact that the upper halves are either all zeros
3250  // or all ones:
3251  //
3252  // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
3253  //
3254  // and grouping the right terms together since they are quicker than the
3255  // multiplication:
3256  //
3257  // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
3258  SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
3259  SDValue LL = Op.getOperand(0);
3260  SDValue RL = Op.getOperand(1);
3261  SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
3262  SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
3263  // SystemZISD::UMUL_LOHI returns the low result in the odd register and
3264  // the high result in the even register. ISD::SMUL_LOHI is defined to
3265  // return the low half first, so the results are in reverse order.
3267  LL, RL, Ops[1], Ops[0]);
3268  SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
3269  SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
3270  SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
3271  Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum);
3272  }
3273  return DAG.getMergeValues(Ops, DL);
3274 }
3275 
3276 SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
3277  SelectionDAG &DAG) const {
3278  EVT VT = Op.getValueType();
3279  SDLoc DL(Op);
3280  SDValue Ops[2];
3281  if (is32Bit(VT))
3282  // Just do a normal 64-bit multiplication and extract the results.
3283  // We define this so that it can be used for constant division.
3285  Op.getOperand(1), Ops[1], Ops[0]);
3286  else
3287  // SystemZISD::UMUL_LOHI returns the low result in the odd register and
3288  // the high result in the even register. ISD::UMUL_LOHI is defined to
3289  // return the low half first, so the results are in reverse order.
3291  Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
3292  return DAG.getMergeValues(Ops, DL);
3293 }
3294 
3295 SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
3296  SelectionDAG &DAG) const {
3297  SDValue Op0 = Op.getOperand(0);
3298  SDValue Op1 = Op.getOperand(1);
3299  EVT VT = Op.getValueType();
3300  SDLoc DL(Op);
3301 
3302  // We use DSGF for 32-bit division. This means the first operand must
3303  // always be 64-bit, and the second operand should be 32-bit whenever
3304  // that is possible, to improve performance.
3305  if (is32Bit(VT))
3306  Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0);
3307  else if (DAG.ComputeNumSignBits(Op1) > 32)
3308  Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
3309 
3310  // DSG(F) returns the remainder in the even register and the
3311  // quotient in the odd register.
3312  SDValue Ops[2];
3313  lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]);
3314  return DAG.getMergeValues(Ops, DL);
3315 }
3316 
3317 SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
3318  SelectionDAG &DAG) const {
3319  EVT VT = Op.getValueType();
3320  SDLoc DL(Op);
3321 
3322  // DL(G) returns the remainder in the even register and the
3323  // quotient in the odd register.
3324  SDValue Ops[2];
3326  Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
3327  return DAG.getMergeValues(Ops, DL);
3328 }
3329 
3330 SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
3331  assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
3332 
3333  // Get the known-zero masks for each operand.
3334  SDValue Ops[] = {Op.getOperand(0), Op.getOperand(1)};
3335  KnownBits Known[2] = {DAG.computeKnownBits(Ops[0]),
3336  DAG.computeKnownBits(Ops[1])};
3337 
3338  // See if the upper 32 bits of one operand and the lower 32 bits of the
3339  // other are known zero. They are the low and high operands respectively.
3340  uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
3341  Known[1].Zero.getZExtValue() };
3342  unsigned High, Low;
3343  if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
3344  High = 1, Low = 0;
3345  else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
3346  High = 0, Low = 1;
3347  else
3348  return Op;
3349 
3350  SDValue LowOp = Ops[Low];
3351  SDValue HighOp = Ops[High];
3352 
3353  // If the high part is a constant, we're better off using IILH.
3354  if (HighOp.getOpcode() == ISD::Constant)
3355  return Op;
3356 
3357  // If the low part is a constant that is outside the range of LHI,
3358  // then we're better off using IILF.
3359  if (LowOp.getOpcode() == ISD::Constant) {
3360  int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue());
3361  if (!isInt<16>(Value))
3362  return Op;
3363  }
3364 
3365  // Check whether the high part is an AND that doesn't change the
3366  // high 32 bits and just masks out low bits. We can skip it if so.
3367  if (HighOp.getOpcode() == ISD::AND &&
3368  HighOp.getOperand(1).getOpcode() == ISD::Constant) {
3369  SDValue HighOp0 = HighOp.getOperand(0);
3370  uint64_t Mask = cast<ConstantSDNode>(HighOp.getOperand(1))->getZExtValue();
3371  if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff))))
3372  HighOp = HighOp0;
3373  }
3374 
3375  // Take advantage of the fact that all GR32 operations only change the
3376  // low 32 bits by truncating Low to an i32 and inserting it directly
3377  // using a subreg. The interesting cases are those where the truncation
3378  // can be folded.
3379  SDLoc DL(Op);
3380  SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
3381  return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
3382  MVT::i64, HighOp, Low32);
3383 }
3384 
3385 // Lower SADDO/SSUBO/UADDO/USUBO nodes.
3386 SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
3387  SelectionDAG &DAG) const {
3388  SDNode *N = Op.getNode();
3389  SDValue LHS = N->getOperand(0);
3390  SDValue RHS = N->getOperand(1);
3391  SDLoc DL(N);
3392  unsigned BaseOp = 0;
3393  unsigned CCValid = 0;
3394  unsigned CCMask = 0;
3395 
3396  switch (Op.getOpcode()) {
3397  default: llvm_unreachable("Unknown instruction!");
3398  case ISD::SADDO:
3399  BaseOp = SystemZISD::SADDO;
3400  CCValid = SystemZ::CCMASK_ARITH;
3402  break;
3403  case ISD::SSUBO:
3404  BaseOp = SystemZISD::SSUBO;
3405  CCValid = SystemZ::CCMASK_ARITH;
3407  break;
3408  case ISD::UADDO:
3409  BaseOp = SystemZISD::UADDO;
3410  CCValid = SystemZ::CCMASK_LOGICAL;
3412  break;
3413  case ISD::USUBO:
3414  BaseOp = SystemZISD::USUBO;
3415  CCValid = SystemZ::CCMASK_LOGICAL;
3417  break;
3418  }
3419 
3420  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
3421  SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
3422 
3423  SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
3424  if (N->getValueType(1) == MVT::i1)
3425  SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
3426 
3427  return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
3428 }
3429 
3430 // Lower ADDCARRY/SUBCARRY nodes.
3431 SDValue SystemZTargetLowering::lowerADDSUBCARRY(SDValue Op,
3432  SelectionDAG &DAG) const {
3433 
3434  SDNode *N = Op.getNode();
3435  MVT VT = N->getSimpleValueType(0);
3436 
3437  // Let legalize expand this if it isn't a legal type yet.
3438  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
3439  return SDValue();
3440 
3441  SDValue LHS = N->getOperand(0);
3442  SDValue RHS = N->getOperand(1);
3443  SDValue Carry = Op.getOperand(2);
3444  SDLoc DL(N);
3445  unsigned BaseOp = 0;
3446  unsigned CCValid = 0;
3447  unsigned CCMask = 0;
3448 
3449  switch (Op.getOpcode()) {
3450  default: llvm_unreachable("Unknown instruction!");
3451  case ISD::ADDCARRY:
3452  BaseOp = SystemZISD::ADDCARRY;
3453  CCValid = SystemZ::CCMASK_LOGICAL;
3455  break;
3456  case ISD::SUBCARRY:
3457  BaseOp = SystemZISD::SUBCARRY;
3458  CCValid = SystemZ::CCMASK_LOGICAL;
3460  break;
3461  }
3462 
3463  // Set the condition code from the carry flag.
3464  Carry = DAG.getNode(SystemZISD::GET_CCMASK, DL, MVT::i32, Carry,
3465  DAG.getConstant(CCValid, DL, MVT::i32),
3466  DAG.getConstant(CCMask, DL, MVT::i32));
3467 
3468  SDVTList VTs = DAG.getVTList(VT, MVT::i32);
3469  SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry);
3470 
3471  SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
3472  if (N->getValueType(1) == MVT::i1)
3473  SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
3474 
3475  return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
3476 }
3477 
3478 SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
3479  SelectionDAG &DAG) const {
3480  EVT VT = Op.getValueType();
3481  SDLoc DL(Op);
3482  Op = Op.getOperand(0);
3483 
3484  // Handle vector types via VPOPCT.
3485  if (VT.isVector()) {
3486  Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
3487  Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
3488  switch (VT.getScalarSizeInBits()) {
3489  case 8:
3490  break;
3491  case 16: {
3492  Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
3493  SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
3494  SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
3495  Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
3496  Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
3497  break;
3498  }
3499  case 32: {
3500  SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
3501  DAG.getConstant(0, DL, MVT::i32));
3502  Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
3503  break;
3504  }
3505  case 64: {
3506  SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
3507  DAG.getConstant(0, DL, MVT::i32));
3508  Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
3509  Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
3510  break;
3511  }
3512  default:
3513  llvm_unreachable("Unexpected type");
3514  }
3515  return Op;
3516  }
3517 
3518  // Get the known-zero mask for the operand.
3519  KnownBits Known = DAG.computeKnownBits(Op);
3520  unsigned NumSignificantBits = (~Known.Zero).getActiveBits();
3521  if (NumSignificantBits == 0)
3522  return DAG.getConstant(0, DL, VT);
3523 
3524  // Skip known-zero high parts of the operand.
3525  int64_t OrigBitSize = VT.getSizeInBits();
3526  int64_t BitSize = (int64_t)1 << Log2_32_Ceil(NumSignificantBits);
3527  BitSize = std::min(BitSize, OrigBitSize);
3528 
3529  // The POPCNT instruction counts the number of bits in each byte.
3530  Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op);
3531  Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op);
3532  Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
3533 
3534  // Add up per-byte counts in a binary tree. All bits of Op at
3535  // position larger than BitSize remain zero throughout.
3536  for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
3537  SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
3538  if (BitSize != OrigBitSize)
3539  Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
3540  DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
3541  Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
3542  }
3543 
3544  // Extract overall result from high byte.
3545  if (BitSize > 8)
3546  Op = DAG.getNode(ISD::SRL, DL, VT, Op,
3547  DAG.getConstant(BitSize - 8, DL, VT));
3548 
3549  return Op;
3550 }
3551 
3552 SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
3553  SelectionDAG &DAG) const {
3554  SDLoc DL(Op);
3555  AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
3556  cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
3557  SyncScope::ID FenceSSID = static_cast<SyncScope::ID>(
3558  cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
3559 
3560  // The only fence that needs an instruction is a sequentially-consistent
3561  // cross-thread fence.
3562  if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
3563  FenceSSID == SyncScope::System) {
3564  return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
3565  Op.getOperand(0)),
3566  0);
3567  }
3568 
3569  // MEMBARRIER is a compiler barrier; it codegens to a no-op.
3570  return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
3571 }
3572 
3573 // Op is an atomic load. Lower it into a normal volatile load.
3574 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
3575  SelectionDAG &DAG) const {
3576  auto *Node = cast<AtomicSDNode>(Op.getNode());
3577  return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
3578  Node->getChain(), Node->getBasePtr(),
3579  Node->getMemoryVT(), Node->getMemOperand());
3580 }
3581 
3582 // Op is an atomic store. Lower it into a normal volatile store.
3583 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
3584  SelectionDAG &DAG) const {
3585  auto *Node = cast<AtomicSDNode>(Op.getNode());
3586  SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
3587  Node->getBasePtr(), Node->getMemoryVT(),
3588  Node->getMemOperand());
3589  // We have to enforce sequential consistency by performing a
3590  // serialization operation after the store.
3591  if (Node->getOrdering() == AtomicOrdering::SequentiallyConsistent)
3592  Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
3593  MVT::Other, Chain), 0);
3594  return Chain;
3595 }
3596 
3597 // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
3598 // two into the fullword ATOMIC_LOADW_* operation given by Opcode.
3599 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
3600  SelectionDAG &DAG,
3601  unsigned Opcode) const {
3602  auto *Node = cast<AtomicSDNode>(Op.getNode());
3603 
3604  // 32-bit operations need no code outside the main loop.
3605  EVT NarrowVT = Node->getMemoryVT();
3606  EVT WideVT = MVT::i32;
3607  if (NarrowVT == WideVT)
3608  return Op;
3609 
3610  int64_t BitSize = NarrowVT.getSizeInBits();
3611  SDValue ChainIn = Node->getChain();
3612  SDValue Addr = Node->getBasePtr();
3613  SDValue Src2 = Node->getVal();
3614  MachineMemOperand *MMO = Node->getMemOperand();
3615  SDLoc DL(Node);
3616  EVT PtrVT = Addr.getValueType();
3617 
3618  // Convert atomic subtracts of constants into additions.
3619  if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
3620  if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
3622  Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType());
3623  }
3624 
3625  // Get the address of the containing word.
3626  SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3627  DAG.getConstant(-4, DL, PtrVT));
3628 
3629  // Get the number of bits that the word must be rotated left in order
3630  // to bring the field to the top bits of a GR32.
3631  SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3632  DAG.getConstant(3, DL, PtrVT));
3633  BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
3634 
3635  // Get the complementing shift amount, for rotating a field in the top
3636  // bits back to its proper position.
3637  SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3638  DAG.getConstant(0, DL, WideVT), BitShift);
3639 
3640  // Extend the source operand to 32 bits and prepare it for the inner loop.
3641  // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
3642  // operations require the source to be shifted in advance. (This shift
3643  // can be folded if the source is constant.) For AND and NAND, the lower
3644  // bits must be set, while for other opcodes they should be left clear.
3645  if (Opcode != SystemZISD::ATOMIC_SWAPW)
3646  Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
3647  DAG.getConstant(32 - BitSize, DL, WideVT));
3648  if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
3650  Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
3651  DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
3652 
3653  // Construct the ATOMIC_LOADW_* node.
3654  SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
3655  SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
3656  DAG.getConstant(BitSize, DL, WideVT) };
3657  SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
3658  NarrowVT, MMO);
3659 
3660  // Rotate the result of the final CS so that the field is in the lower
3661  // bits of a GR32, then truncate it.
3662  SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
3663  DAG.getConstant(BitSize, DL, WideVT));
3664  SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
3665 
3666  SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
3667  return DAG.getMergeValues(RetOps, DL);
3668 }
3669 
3670 // Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations
3671 // into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit
3672 // operations into additions.
3673 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
3674  SelectionDAG &DAG) const {
3675  auto *Node = cast<AtomicSDNode>(Op.getNode());
3676  EVT MemVT = Node->getMemoryVT();
3677  if (MemVT == MVT::i32 || MemVT == MVT::i64) {
3678  // A full-width operation.
3679  assert(Op.getValueType() == MemVT && "Mismatched VTs");
3680  SDValue Src2 = Node->getVal();
3681  SDValue NegSrc2;
3682  SDLoc DL(Src2);
3683 
3684  if (auto *Op2 = dyn_cast<ConstantSDNode>(Src2)) {
3685  // Use an addition if the operand is constant and either LAA(G) is
3686  // available or the negative value is in the range of A(G)FHI.
3687  int64_t Value = (-Op2->getAPIntValue()).getSExtValue();
3688  if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1())
3689  NegSrc2 = DAG.getConstant(Value, DL, MemVT);
3690  } else if (Subtarget.hasInterlockedAccess1())
3691  // Use LAA(G) if available.
3692  NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT),
3693  Src2);
3694 
3695  if (NegSrc2.getNode())
3696  return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
3697  Node->getChain(), Node->getBasePtr(), NegSrc2,
3698  Node->getMemOperand());
3699 
3700  // Use the node as-is.
3701  return Op;
3702  }
3703 
3704  return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
3705 }
3706 
3707 // Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
3708 SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
3709  SelectionDAG &DAG) const {
3710  auto *Node = cast<AtomicSDNode>(Op.getNode());
3711  SDValue ChainIn = Node->getOperand(0);
3712  SDValue Addr = Node->getOperand(1);
3713  SDValue CmpVal = Node->getOperand(2);
3714  SDValue SwapVal = Node->getOperand(3);
3715  MachineMemOperand *MMO = Node->getMemOperand();
3716  SDLoc DL(Node);
3717 
3718  // We have native support for 32-bit and 64-bit compare and swap, but we
3719  // still need to expand extracting the "success" result from the CC.
3720  EVT NarrowVT = Node->getMemoryVT();
3721  EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
3722  if (NarrowVT == WideVT) {
3723  SDVTList Tys = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
3724  SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
3726  DL, Tys, Ops, NarrowVT, MMO);
3727  SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
3729 
3730  DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
3732  DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
3733  return SDValue();
3734  }
3735 
3736  // Convert 8-bit and 16-bit compare and swap to a loop, implemented
3737  // via a fullword ATOMIC_CMP_SWAPW operation.
3738  int64_t BitSize = NarrowVT.getSizeInBits();
3739  EVT PtrVT = Addr.getValueType();
3740 
3741  // Get the address of the containing word.
3742  SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3743  DAG.getConstant(-4, DL, PtrVT));
3744 
3745  // Get the number of bits that the word must be rotated left in order
3746  // to bring the field to the top bits of a GR32.
3747  SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3748  DAG.getConstant(3, DL, PtrVT));
3749  BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
3750 
3751  // Get the complementing shift amount, for rotating a field in the top
3752  // bits back to its proper position.
3753  SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3754  DAG.getConstant(0, DL, WideVT), BitShift);
3755 
3756  // Construct the ATOMIC_CMP_SWAPW node.
3757  SDVTList VTList = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
3758  SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
3759  NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
3761  VTList, Ops, NarrowVT, MMO);
3762  SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
3764 
3765  DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
3767  DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
3768  return SDValue();
3769 }
3770 
3772 SystemZTargetLowering::getMMOFlags(const Instruction &I) const {
3773  // Because of how we convert atomic_load and atomic_store to normal loads and
3774  // stores in the DAG, we need to ensure that the MMOs are marked volatile
3775  // since DAGCombine hasn't been updated to account for atomic, but non
3776  // volatile loads. (See D57601)
3777  if (auto *SI = dyn_cast<StoreInst>(&I))
3778  if (SI->isAtomic())
3780  if (auto *LI = dyn_cast<LoadInst>(&I))
3781  if (LI->isAtomic())
3783  if (auto *AI = dyn_cast<AtomicRMWInst>(&I))
3784  if (AI->isAtomic())
3786  if (auto *AI = dyn_cast<AtomicCmpXchgInst>(&I))
3787  if (AI->isAtomic())
3790 }
3791 
3792 SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
3793  SelectionDAG &DAG) const {
3794  MachineFunction &MF = DAG.getMachineFunction();
3795  MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true);
3796  return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op),
3797  SystemZ::R15D, Op.getValueType());
3798 }
3799 
3800 SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
3801  SelectionDAG &DAG) const {
3802  MachineFunction &MF = DAG.getMachineFunction();
3803  MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true);
3804  bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
3805 
3806  SDValue Chain = Op.getOperand(0);
3807  SDValue NewSP = Op.getOperand(1);
3808  SDValue Backchain;
3809  SDLoc DL(Op);
3810 
3811  if (StoreBackchain) {
3812  SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, MVT::i64);
3813  Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo());
3814  }
3815 
3816  Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R15D, NewSP);
3817 
3818  if (StoreBackchain)
3819  Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo());
3820 
3821  return Chain;
3822 }
3823 
3824 SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
3825  SelectionDAG &DAG) const {
3826  bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
3827  if (!IsData)
3828  // Just preserve the chain.
3829  return Op.getOperand(0);
3830 
3831  SDLoc DL(Op);
3832  bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
3833  unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
3834  auto *Node = cast<MemIntrinsicSDNode>(Op.getNode());
3835  SDValue Ops[] = {
3836  Op.getOperand(0),
3837  DAG.getConstant(Code, DL, MVT::i32),
3838  Op.getOperand(1)
3839  };
3841  Node->getVTList(), Ops,
3842  Node->getMemoryVT(), Node->getMemOperand());
3843 }
3844 
3845 // Convert condition code in CCReg to an i32 value.
3847  SDLoc DL(CCReg);
3848  SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
3849  return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
3851 }
3852 
3853 SDValue
3854 SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
3855  SelectionDAG &DAG) const {
3856  unsigned Opcode, CCValid;
3857  if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
3858  assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
3859  SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
3860  SDValue CC = getCCResult(DAG, SDValue(Node, 0));
3861  DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), CC);
3862  return SDValue();
3863  }
3864 
3865  return SDValue();
3866 }
3867 
3868 SDValue
3869 SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
3870  SelectionDAG &DAG) const {
3871  unsigned Opcode, CCValid;
3872  if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
3873  SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
3874  if (Op->getNumValues() == 1)
3875  return getCCResult(DAG, SDValue(Node, 0));
3876  assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
3877  return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(),
3878  SDValue(Node, 0), getCCResult(DAG, SDValue(Node, 1)));
3879  }
3880 
3881  unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3882  switch (Id) {
3883  case Intrinsic::thread_pointer:
3884  return lowerThreadPointer(SDLoc(Op), DAG);
3885 
3886  case Intrinsic::s390_vpdi:
3887  return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(),
3888  Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3889 
3890  case Intrinsic::s390_vperm:
3891  return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(),
3892  Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3893 
3894  case Intrinsic::s390_vuphb:
3895  case Intrinsic::s390_vuphh:
3896  case Intrinsic::s390_vuphf:
3897  return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(),
3898  Op.getOperand(1));
3899 
3900  case Intrinsic::s390_vuplhb:
3901  case Intrinsic::s390_vuplhh:
3902  case Intrinsic::s390_vuplhf:
3903  return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(),
3904  Op.getOperand(1));
3905 
3906  case Intrinsic::s390_vuplb:
3907  case Intrinsic::s390_vuplhw:
3908  case Intrinsic::s390_vuplf:
3909  return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(),
3910  Op.getOperand(1));
3911 
3912  case Intrinsic::s390_vupllb:
3913  case Intrinsic::s390_vupllh:
3914  case Intrinsic::s390_vupllf:
3915  return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(),
3916  Op.getOperand(1));
3917 
3918  case Intrinsic::s390_vsumb:
3919  case Intrinsic::s390_vsumh:
3920  case Intrinsic::s390_vsumgh:
3921  case Intrinsic::s390_vsumgf:
3922  case Intrinsic::s390_vsumqf:
3923  case Intrinsic::s390_vsumqg:
3924  return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(),
3925  Op.getOperand(1), Op.getOperand(2));
3926  }
3927 
3928  return SDValue();
3929 }
3930 
3931 namespace {
3932 // Says that SystemZISD operation Opcode can be used to perform the equivalent
3933 // of a VPERM with permute vector Bytes. If Opcode takes three operands,
3934 // Operand is the constant third operand, otherwise it is the number of
3935 // bytes in each element of the result.
3936 struct Permute {
3937  unsigned Opcode;
3938  unsigned Operand;
3939  unsigned char Bytes[SystemZ::VectorBytes];
3940 };
3941 }
3942 
3943 static const Permute PermuteForms[] = {
3944  // VMRHG
3946  { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
3947  // VMRHF
3949  { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
3950  // VMRHH
3952  { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
3953  // VMRHB
3955  { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
3956  // VMRLG
3957  { SystemZISD::MERGE_LOW, 8,
3958  { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
3959  // VMRLF
3960  { SystemZISD::MERGE_LOW, 4,
3961  { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
3962  // VMRLH
3963  { SystemZISD::MERGE_LOW, 2,
3964  { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
3965  // VMRLB
3966  { SystemZISD::MERGE_LOW, 1,
3967  { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
3968  // VPKG
3969  { SystemZISD::PACK, 4,
3970  { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
3971  // VPKF
3972  { SystemZISD::PACK, 2,
3973  { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
3974  // VPKH
3975  { SystemZISD::PACK, 1,
3976  { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
3977  // VPDI V1, V2, 4 (low half of V1, high half of V2)
3979  { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
3980  // VPDI V1, V2, 1 (high half of V1, low half of V2)
3982  { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
3983 };
3984 
3985 // Called after matching a vector shuffle against a particular pattern.
3986 // Both the original shuffle and the pattern have two vector operands.
3987 // OpNos[0] is the operand of the original shuffle that should be used for
3988 // operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
3989 // OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
3990 // set OpNo0 and OpNo1 to the shuffle operands that should actually be used
3991 // for operands 0 and 1 of the pattern.
3992 static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
3993  if (OpNos[0] < 0) {
3994  if (OpNos[1] < 0)
3995  return false;
3996  OpNo0 = OpNo1 = OpNos[1];
3997  } else if (OpNos[1] < 0) {
3998  OpNo0 = OpNo1 = OpNos[0];
3999  } else {
4000  OpNo0 = OpNos[0];
4001  OpNo1 = OpNos[1];
4002  }
4003  return true;
4004 }
4005 
4006 // Bytes is a VPERM-like permute vector, except that -1 is used for
4007 // undefined bytes. Return true if the VPERM can be implemented using P.
4008 // When returning true set OpNo0 to the VPERM operand that should be
4009 // used for operand 0 of P and likewise OpNo1 for operand 1 of P.
4010 //
4011 // For example, if swapping the VPERM operands allows P to match, OpNo0
4012 // will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
4013 // operand, but rewriting it to use two duplicated operands allows it to
4014 // match P, then OpNo0 and OpNo1 will be the same.
4015 static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
4016  unsigned &OpNo0, unsigned &OpNo1) {
4017  int OpNos[] = { -1, -1 };
4018  for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
4019  int Elt = Bytes[I];
4020  if (Elt >= 0) {
4021  // Make sure that the two permute vectors use the same suboperand
4022  // byte number. Only the operand numbers (the high bits) are
4023  // allowed to differ.
4024  if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
4025  return false;
4026  int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
4027  int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
4028  // Make sure that the operand mappings are consistent with previous
4029  // elements.
4030  if (OpNos[ModelOpNo] == 1 - RealOpNo)
4031  return false;
4032  OpNos[ModelOpNo] = RealOpNo;
4033  }
4034  }
4035  return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
4036 }
4037 
4038 // As above, but search for a matching permute.
4039 static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
4040  unsigned &OpNo0, unsigned &OpNo1) {
4041  for (auto &P : PermuteForms)
4042  if (matchPermute(Bytes, P, OpNo0, OpNo1))
4043  return &P;
4044  return nullptr;
4045 }
4046 
4047 // Bytes is a VPERM-like permute vector, except that -1 is used for
4048 // undefined bytes. This permute is an operand of an outer permute.
4049 // See whether redistributing the -1 bytes gives a shuffle that can be
4050 // implemented using P. If so, set Transform to a VPERM-like permute vector
4051 // that, when applied to the result of P, gives the original permute in Bytes.
4052 static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
4053  const Permute &P,
4054  SmallVectorImpl<int> &Transform) {
4055  unsigned To = 0;
4056  for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
4057  int Elt = Bytes[From];
4058  if (Elt < 0)
4059  // Byte number From of the result is undefined.
4060  Transform[From] = -1;
4061  else {
4062  while (P.Bytes[To] != Elt) {
4063  To += 1;
4064  if (To == SystemZ::VectorBytes)
4065  return false;
4066  }
4067  Transform[From] = To;
4068  }
4069  }
4070  return true;
4071 }
4072 
4073 // As above, but search for a matching permute.
4074 static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
4075  SmallVectorImpl<int> &Transform) {
4076  for (auto &P : PermuteForms)
4077  if (matchDoublePermute(Bytes, P, Transform))
4078  return &P;
4079  return nullptr;
4080 }
4081 
4082 // Convert the mask of the given shuffle op into a byte-level mask,
4083 // as if it had type vNi8.
4084 static bool getVPermMask(SDValue ShuffleOp,
4085  SmallVectorImpl<int> &Bytes) {
4086  EVT VT = ShuffleOp.getValueType();
4087  unsigned NumElements = VT.getVectorNumElements();
4088  unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
4089 
4090  if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(ShuffleOp)) {
4091  Bytes.resize(NumElements * BytesPerElement, -1);
4092  for (unsigned I = 0; I < NumElements; ++I) {
4093  int Index = VSN->getMaskElt(I);
4094  if (Index >= 0)
4095  for (unsigned J = 0; J < BytesPerElement; ++J)
4096  Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
4097  }
4098  return true;
4099  }
4100  if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
4101  isa<ConstantSDNode>(ShuffleOp.getOperand(1))) {
4102  unsigned Index = ShuffleOp.getConstantOperandVal(1);
4103  Bytes.resize(NumElements * BytesPerElement, -1);
4104  for (unsigned I = 0; I < NumElements; ++I)
4105  for (unsigned J = 0; J < BytesPerElement; ++J)
4106  Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
4107  return true;
4108  }
4109  return false;
4110 }
4111 
4112 // Bytes is a VPERM-like permute vector, except that -1 is used for
4113 // undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
4114 // the result come from a contiguous sequence of bytes from one input.
4115 // Set Base to the selector for the first byte if so.
4116 static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
4117  unsigned BytesPerElement, int &Base) {
4118  Base = -1;
4119  for (unsigned I = 0; I < BytesPerElement; ++I) {
4120  if (Bytes[Start + I] >= 0) {
4121  unsigned Elem = Bytes[Start + I];
4122  if (Base < 0) {
4123  Base = Elem - I;
4124  // Make sure the bytes would come from one input operand.
4125  if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
4126  return false;
4127  } else if (unsigned(Base) != Elem - I)
4128  return false;
4129  }
4130  }
4131  return true;
4132 }
4133 
4134 // Bytes is a VPERM-like permute vector, except that -1 is used for
4135 // undefined bytes. Return true if it can be performed using VSLDI.
4136 // When returning true, set StartIndex to the shift amount and OpNo0
4137 // and OpNo1 to the VPERM operands that should be used as the first
4138 // and second shift operand respectively.
4139 static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
4140  unsigned &StartIndex, unsigned &OpNo0,
4141  unsigned &OpNo1) {
4142  int OpNos[] = { -1, -1 };
4143  int Shift = -1;
4144  for (unsigned I = 0; I < 16; ++I) {
4145  int Index = Bytes[I];
4146  if (Index >= 0) {
4147  int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
4148  int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
4149  int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
4150  if (Shift < 0)
4151  Shift = ExpectedShift;
4152  else if (Shift != ExpectedShift)
4153  return false;
4154  // Make sure that the operand mappings are consistent with previous
4155  // elements.
4156  if (OpNos[ModelOpNo] == 1 - RealOpNo)
4157  return false;
4158  OpNos[ModelOpNo] = RealOpNo;
4159  }
4160  }
4161  StartIndex = Shift;
4162  return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
4163 }
4164 
4165 // Create a node that performs P on operands Op0 and Op1, casting the
4166 // operands to the appropriate type. The type of the result is determined by P.
4167 static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
4168  const Permute &P, SDValue Op0, SDValue Op1) {
4169  // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
4170  // elements of a PACK are twice as wide as the outputs.
4171  unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
4172  P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
4173  P.Operand);
4174  // Cast both operands to the appropriate type.
4175  MVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBytes * 8),
4176  SystemZ::VectorBytes / InBytes);
4177  Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
4178  Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
4179  SDValue Op;
4180  if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
4181  SDValue Op2 = DAG.getConstant(P.Operand, DL, MVT::i32);
4182  Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
4183  } else if (P.Opcode == SystemZISD::PACK) {
4184  MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8),
4185  SystemZ::VectorBytes / P.Operand);
4186  Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
4187  } else {
4188  Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
4189  }
4190  return Op;
4191 }
4192 
4193 // Bytes is a VPERM-like permute vector, except that -1 is used for
4194 // undefined bytes. Implement it on operands Ops[0] and Ops[1] using
4195 // VSLDI or VPERM.
4197  SDValue *Ops,
4198  const SmallVectorImpl<int> &Bytes) {
4199  for (unsigned I = 0; I < 2; ++I)
4200  Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
4201 
4202  // First see whether VSLDI can be used.
4203  unsigned StartIndex, OpNo0, OpNo1;
4204  if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
4205  return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
4206  Ops[OpNo1], DAG.getConstant(StartIndex, DL, MVT::i32));
4207 
4208  // Fall back on VPERM. Construct an SDNode for the permute vector.
4209  SDValue IndexNodes[SystemZ::VectorBytes];
4210  for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
4211  if (Bytes[I] >= 0)
4212  IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
4213  else
4214  IndexNodes[I] = DAG.getUNDEF(MVT::i32);
4215  SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
4216  return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0], Ops[1], Op2);
4217 }
4218 
4219 namespace {
4220 // Describes a general N-operand vector shuffle.
4221 struct GeneralShuffle {
4222  GeneralShuffle(EVT vt) : VT(vt) {}
4223  void addUndef();
4224  bool add(SDValue, unsigned);
4225  SDValue getNode(SelectionDAG &, const SDLoc &);
4226 
4227  // The operands of the shuffle.
4229 
4230  // Index I is -1 if byte I of the result is undefined. Otherwise the
4231  // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
4232  // Bytes[I] / SystemZ::VectorBytes.
4234 
4235  // The type of the shuffle result.
4236  EVT VT;
4237 };
4238 }
4239 
4240 // Add an extra undefined element to the shuffle.
4241 void GeneralShuffle::addUndef() {
4242  unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
4243  for (unsigned I = 0; I < BytesPerElement; ++I)
4244  Bytes.push_back(-1);
4245 }
4246 
4247 // Add an extra element to the shuffle, taking it from element Elem of Op.
4248 // A null Op indicates a vector input whose value will be calculated later;
4249 // there is at most one such input per shuffle and it always has the same
4250 // type as the result. Aborts and returns false if the source vector elements
4251 // of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
4252 // LLVM they become implicitly extended, but this is rare and not optimized.
4253 bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
4254  unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
4255 
4256  // The source vector can have wider elements than the result,
4257  // either through an explicit TRUNCATE or because of type legalization.
4258  // We want the least significant part.
4259  EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
4260  unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
4261 
4262  // Return false if the source elements are smaller than their destination
4263  // elements.
4264  if (FromBytesPerElement < BytesPerElement)
4265  return false;
4266 
4267  unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
4268  (FromBytesPerElement - BytesPerElement));
4269 
4270  // Look through things like shuffles and bitcasts.
4271  while (Op.getNode()) {
4272  if (Op.getOpcode() == ISD::BITCAST)
4273  Op = Op.getOperand(0);
4274  else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
4275  // See whether the bytes we need come from a contiguous part of one
4276  // operand.
4278  if (!getVPermMask(Op, OpBytes))
4279  break;
4280  int NewByte;
4281  if (!getShuffleInput(OpBytes, Byte, BytesPerElement, NewByte))
4282  break;
4283  if (NewByte < 0) {
4284  addUndef();
4285  return true;
4286  }
4287  Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes);
4288  Byte = unsigned(NewByte) % SystemZ::VectorBytes;
4289  } else if (Op.isUndef()) {
4290  addUndef();
4291  return true;
4292  } else
4293  break;
4294  }
4295 
4296  // Make sure that the source of the extraction is in Ops.
4297  unsigned OpNo = 0;
4298  for (; OpNo < Ops.size(); ++OpNo)
4299  if (Ops[OpNo] == Op)
4300  break;
4301  if (OpNo == Ops.size())
4302  Ops.push_back(Op);
4303 
4304  // Add the element to Bytes.
4305  unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
4306  for (unsigned I = 0; I < BytesPerElement; ++I)
4307  Bytes.push_back(Base + I);
4308 
4309  return true;
4310 }
4311 
4312 // Return SDNodes for the completed shuffle.
4313 SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
4314  assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
4315 
4316  if (Ops.size() == 0)
4317  return DAG.getUNDEF(VT);
4318 
4319  // Make sure that there are at least two shuffle operands.
4320  if (Ops.size() == 1)
4321  Ops.push_back(DAG.getUNDEF(MVT::v16i8));
4322 
4323  // Create a tree of shuffles, deferring root node until after the loop.
4324  // Try to redistribute the undefined elements of non-root nodes so that
4325  // the non-root shuffles match something like a pack or merge, then adjust
4326  // the parent node's permute vector to compensate for the new order.
4327  // Among other things, this copes with vectors like <2 x i16> that were
4328  // padded with undefined elements during type legalization.
4329  //
4330  // In the best case this redistribution will lead to the whole tree
4331  // using packs and merges. It should rarely be a loss in other cases.
4332  unsigned Stride = 1;
4333  for (; Stride * 2 < Ops.size(); Stride *= 2) {
4334  for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
4335  SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
4336 
4337  // Create a mask for just these two operands.
4339  for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
4340  unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
4341  unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
4342  if (OpNo == I)
4343  NewBytes[J] = Byte;
4344  else if (OpNo == I + Stride)
4345  NewBytes[J] = SystemZ::VectorBytes + Byte;
4346  else
4347  NewBytes[J] = -1;
4348  }
4349  // See if it would be better to reorganize NewMask to avoid using VPERM.
4350  SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
4351  if (const Permute *P = matchDoublePermute(NewBytes, NewBytesMap)) {
4352  Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
4353  // Applying NewBytesMap to Ops[I] gets back to NewBytes.
4354  for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
4355  if (NewBytes[J] >= 0) {
4356  assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
4357  "Invalid double permute");
4358  Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
4359  } else
4360  assert(NewBytesMap[J] < 0 && "Invalid double permute");
4361  }
4362  } else {
4363  // Just use NewBytes on the operands.
4364  Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
4365  for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
4366  if (NewBytes[J] >= 0)
4367  Bytes[J] = I * SystemZ::VectorBytes + J;
4368  }
4369  }
4370  }
4371 
4372  // Now we just have 2 inputs. Put the second operand in Ops[1].
4373  if (Stride > 1) {
4374  Ops[1] = Ops[Stride];
4375  for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
4376  if (Bytes[I] >= int(SystemZ::VectorBytes))
4377  Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
4378  }
4379 
4380  // Look for an instruction that can do the permute without resorting
4381  // to VPERM.
4382  unsigned OpNo0, OpNo1;
4383  SDValue Op;
4384  if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
4385  Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
4386  else
4387  Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
4388  return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4389 }
4390 
4391 // Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
4392 static bool isScalarToVector(SDValue Op) {
4393  for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
4394  if (!Op.getOperand(I).isUndef())
4395  return false;
4396  return true;
4397 }
4398 
4399 // Return a vector of type VT that contains Value in the first element.
4400 // The other elements don't matter.
4401 static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4402  SDValue Value) {
4403  // If we have a constant, replicate it to all elements and let the
4404  // BUILD_VECTOR lowering take care of it.
4405  if (Value.getOpcode() == ISD::Constant ||
4406  Value.getOpcode() == ISD::ConstantFP) {
4408  return DAG.getBuildVector(VT, DL, Ops);
4409  }
4410  if (Value.isUndef())
4411  return DAG.getUNDEF(VT);
4412  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
4413 }
4414 
4415 // Return a vector of type VT in which Op0 is in element 0 and Op1 is in
4416 // element 1. Used for cases in which replication is cheap.
4417 static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4418  SDValue Op0, SDValue Op1) {
4419  if (Op0.isUndef()) {
4420  if (Op1.isUndef())
4421  return DAG.getUNDEF(VT);
4422  return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
4423  }
4424  if (Op1.isUndef())
4425  return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
4426  return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
4427  buildScalarToVector(DAG, DL, VT, Op0),
4428  buildScalarToVector(DAG, DL, VT, Op1));
4429 }
4430 
4431 // Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
4432 // vector for them.
4433 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
4434  SDValue Op1) {
4435  if (Op0.isUndef() && Op1.isUndef())
4436  return DAG.getUNDEF(MVT::v2i64);
4437  // If one of the two inputs is undefined then replicate the other one,
4438  // in order to avoid using another register unnecessarily.
4439  if (Op0.isUndef())
4440  Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
4441  else if (Op1.isUndef())
4442  Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
4443  else {
4444  Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
4445  Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
4446  }
4447  return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
4448 }
4449 
4450 // If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
4451 // better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
4452 // the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
4453 // would benefit from this representation and return it if so.
4455  BuildVectorSDNode *BVN) {
4456  EVT VT = BVN->getValueType(0);
4457  unsigned NumElements = VT.getVectorNumElements();
4458 
4459  // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
4460  // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
4461  // need a BUILD_VECTOR, add an additional placeholder operand for that
4462  // BUILD_VECTOR and store its operands in ResidueOps.
4463  GeneralShuffle GS(VT);
4465  bool FoundOne = false;
4466  for (unsigned I = 0; I < NumElements; ++I) {
4467  SDValue Op = BVN->getOperand(I);
4468  if (Op.getOpcode() == ISD::TRUNCATE)
4469  Op = Op.getOperand(0);
4470  if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4471  Op.getOperand(1).getOpcode() == ISD::Constant) {
4472  unsigned Elem = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4473  if (!GS.add(Op.getOperand(0), Elem))
4474  return SDValue();
4475  FoundOne = true;
4476  } else if (Op.isUndef()) {
4477  GS.addUndef();
4478  } else {
4479  if (!GS.add(SDValue(), ResidueOps.size()))
4480  return SDValue();
4481  ResidueOps.push_back(BVN->getOperand(I));
4482  }
4483  }
4484 
4485  // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
4486  if (!FoundOne)
4487  return SDValue();
4488 
4489  // Create the BUILD_VECTOR for the remaining elements, if any.
4490  if (!ResidueOps.empty()) {
4491  while (ResidueOps.size() < NumElements)
4492  ResidueOps.push_back(DAG.getUNDEF(ResidueOps[0].getValueType()));
4493  for (auto &Op : GS.Ops) {
4494  if (!Op.getNode()) {
4495  Op = DAG.getBuildVector(VT, SDLoc(BVN), ResidueOps);
4496  break;
4497  }
4498  }
4499  }
4500  return GS.getNode(DAG, SDLoc(BVN));
4501 }
4502 
4503 // Combine GPR scalar values Elems into a vector of type VT.
4504 static SDValue buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4505  SmallVectorImpl<SDValue> &Elems) {
4506  // See whether there is a single replicated value.
4507  SDValue Single;
4508  unsigned int NumElements = Elems.size();
4509  unsigned int Count = 0;
4510  for (auto Elem : Elems) {
4511  if (!Elem.isUndef()) {
4512  if (!Single.getNode())
4513  Single = Elem;
4514  else if (Elem != Single) {
4515  Single = SDValue();
4516  break;
4517  }
4518  Count += 1;
4519  }
4520  }
4521  // There are three cases here:
4522  //
4523  // - if the only defined element is a loaded one, the best sequence
4524  // is a replicating load.
4525  //
4526  // - otherwise, if the only defined element is an i64 value, we will
4527  // end up with the same VLVGP sequence regardless of whether we short-cut
4528  // for replication or fall through to the later code.
4529  //
4530  // - otherwise, if the only defined element is an i32 or smaller value,
4531  // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
4532  // This is only a win if the single defined element is used more than once.
4533  // In other cases we're better off using a single VLVGx.
4534  if (Single.getNode() && (Count > 1 || Single.getOpcode() == ISD::LOAD))
4535  return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
4536 
4537  // If all elements are loads, use VLREP/VLEs (below).
4538  bool AllLoads = true;
4539  for (auto Elem : Elems)
4540  if (Elem.getOpcode() != ISD::LOAD || cast<LoadSDNode>(Elem)->isIndexed()) {
4541  AllLoads = false;
4542  break;
4543  }
4544 
4545  // The best way of building a v2i64 from two i64s is to use VLVGP.
4546  if (VT == MVT::v2i64 && !AllLoads)
4547  return joinDwords(DAG, DL, Elems[0], Elems[1]);
4548 
4549  // Use a 64-bit merge high to combine two doubles.
4550  if (VT == MVT::v2f64 && !AllLoads)
4551  return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
4552 
4553  // Build v4f32 values directly from the FPRs:
4554  //
4555  // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
4556  // V V VMRHF
4557  // <ABxx> <CDxx>
4558  // V VMRHG
4559  // <ABCD>
4560  if (VT == MVT::v4f32 && !AllLoads) {
4561  SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
4562  SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
4563  // Avoid unnecessary undefs by reusing the other operand.
4564  if (Op01.isUndef())
4565  Op01 = Op23;
4566  else if (Op23.isUndef())
4567  Op23 = Op01;
4568  // Merging identical replications is a no-op.
4569  if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
4570  return Op01;
4571  Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01);
4572  Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23);
4574  DL, MVT::v2i64, Op01, Op23);
4575  return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4576  }
4577 
4578  // Collect