LLVM 24.0.0git
SystemZISelLowering.cpp
Go to the documentation of this file.
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZCallingConv.h"
17#include "llvm/ADT/SmallSet.h"
22#include "llvm/IR/GlobalAlias.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsS390.h"
26#include "llvm/IR/Module.h"
32#include <cctype>
33#include <optional>
34
35using namespace llvm;
36
37#define DEBUG_TYPE "systemz-lower"
38
39// Temporarily let this be disabled by default until all known problems
40// related to argument extensions are fixed.
42 "argext-abi-check", cl::init(false),
43 cl::desc("Verify that narrow int args are properly extended per the "
44 "SystemZ ABI."));
45
46namespace {
47// Represents information about a comparison.
48struct Comparison {
49 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
50 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
51 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
52
53 // The operands to the comparison.
54 SDValue Op0, Op1;
55
56 // Chain if this is a strict floating-point comparison.
57 SDValue Chain;
58
59 // The opcode that should be used to compare Op0 and Op1.
60 unsigned Opcode;
61
62 // A SystemZICMP value. Only used for integer comparisons.
63 unsigned ICmpType;
64
65 // The mask of CC values that Opcode can produce.
66 unsigned CCValid;
67
68 // The mask of CC values for which the original condition is true.
69 unsigned CCMask;
70};
71} // end anonymous namespace
72
73// Classify VT as either 32 or 64 bit.
74static bool is32Bit(EVT VT) {
75 switch (VT.getSimpleVT().SimpleTy) {
76 case MVT::i32:
77 return true;
78 case MVT::i64:
79 return false;
80 default:
81 llvm_unreachable("Unsupported type");
82 }
83}
84
85// Return a version of MachineOperand that can be safely used before the
86// final use.
88 if (Op.isReg())
89 Op.setIsKill(false);
90 return Op;
91}
92
94 const SystemZSubtarget &STI)
95 : TargetLowering(TM, STI), Subtarget(STI) {
96 MVT PtrVT = MVT::i64;
97
98 auto *Regs = STI.getSpecialRegisters();
99
100 // Set up the register classes.
101 if (Subtarget.hasHighWord())
102 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass);
103 else
104 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass);
105 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
106 if (!useSoftFloat()) {
107 if (Subtarget.hasVector()) {
108 addRegisterClass(MVT::f16, &SystemZ::VR16BitRegClass);
109 addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
110 addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
111 } else {
112 addRegisterClass(MVT::f16, &SystemZ::FP16BitRegClass);
113 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
114 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
115 }
116 if (Subtarget.hasVectorEnhancements1())
117 addRegisterClass(MVT::f128, &SystemZ::VR128BitRegClass);
118 else
119 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass);
120
121 if (Subtarget.hasVector()) {
122 addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass);
123 addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass);
124 addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass);
125 addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass);
126 addRegisterClass(MVT::v8f16, &SystemZ::VR128BitRegClass);
127 addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass);
128 addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass);
129 }
130
131 if (Subtarget.hasVector())
132 addRegisterClass(MVT::i128, &SystemZ::VR128BitRegClass);
133 }
134
135 // Without vector support, model GR128 register pairs as v2i64.
136 if (!Subtarget.hasVector())
137 addRegisterClass(MVT::v2i64, &SystemZ::GR128BitRegClass);
138
139 // Compute derived properties from the register classes
140 computeRegisterProperties(Subtarget.getRegisterInfo());
141
142 // Set up special registers.
143 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
144
145 // TODO: It may be better to default to latency-oriented scheduling, however
146 // LLVM's current latency-oriented scheduler can't handle physreg definitions
147 // such as SystemZ has with CC, so set this to the register-pressure
148 // scheduler, because it can.
150
153
155
156 // Instructions are strings of 2-byte aligned 2-byte values.
158 // For performance reasons we prefer 16-byte alignment.
160
161 // Handle operations that are handled in a similar way for all types.
162 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
163 I <= MVT::LAST_FP_VALUETYPE;
164 ++I) {
166 if (isTypeLegal(VT)) {
167 // Lower SET_CC into an IPM-based sequence.
171
172 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
174
175 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
178 }
179 }
180
181 // Expand jump table branches as address arithmetic followed by an
182 // indirect jump.
184
185 // Expand BRCOND into a BR_CC (see above).
187
188 // Handle integer types except i128.
189 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
190 I <= MVT::LAST_INTEGER_VALUETYPE;
191 ++I) {
193 if (isTypeLegal(VT) && VT != MVT::i128) {
195
196 // Expand individual DIV and REMs into DIVREMs.
203
204 // Support addition/subtraction with overflow.
207
208 // Support addition/subtraction with carry.
211
212 // Support carry in as value rather than glue.
215
216 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
217 // available, or if the operand is constant.
219
220 // Use POPCNT on z196 and above.
221 if (Subtarget.hasPopulationCount())
223 else
225
226 // No special instructions for these.
229
230 // Use *MUL_LOHI where possible instead of MULH*.
235
236 // The fp<=>i32/i64 conversions are all Legal except for f16 and for
237 // unsigned on z10 (only z196 and above have native support for
238 // unsigned conversions).
245 // Handle unsigned 32-bit input types as signed 64-bit types on z10.
246 auto OpAction =
247 (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
248 setOperationAction(Op, VT, OpAction);
249 }
250 }
251 }
252
253 // Handle i128 if legal.
254 if (isTypeLegal(MVT::i128)) {
255 // No special instructions for these.
262
263 // We may be able to use VSLDB/VSLD/VSRD for these.
266
267 // No special instructions for these before z17.
268 if (!Subtarget.hasVectorEnhancements3()) {
278 } else {
279 // Even if we do have a legal 128-bit multiply, we do not
280 // want 64-bit multiply-high operations to use it.
283 }
284
285 // Support addition/subtraction with carry.
290
291 // Use VPOPCT and add up partial results.
293
294 // Additional instructions available with z17.
295 if (Subtarget.hasVectorEnhancements3()) {
296 setOperationAction(ISD::ABS, MVT::i128, Legal);
297
299 MVT::i128, Legal);
300 }
301 }
302
303 // These need custom handling in order to handle the f16 conversions.
312
313 // Lower 8- and 16-bit atomic operations into full i32 operations during
314 // type legalization.
320 {MVT::i8, MVT::i16}, Custom);
321
322 // Whether or not i128 is not a legal type, we need to custom lower
323 // the atomic operations in order to exploit SystemZ instructions.
328
329 // Mark sign/zero extending atomic loads as legal, which will make
330 // DAGCombiner fold extensions into atomic loads if possible.
332 {MVT::i8, MVT::i16, MVT::i32}, Legal);
334 {MVT::i8, MVT::i16}, Legal);
336 MVT::i8, Legal);
337
338 // We can use the CC result of compare-and-swap to implement
339 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
343
345
346 // Traps are legal, as we will convert them to "j .+2".
347 setOperationAction(ISD::TRAP, MVT::Other, Legal);
348
349 // We have native support for a 64-bit CTLZ, via FLOGR.
353
354 // On z17 we have native support for a 64-bit CTTZ.
355 if (Subtarget.hasMiscellaneousExtensions4()) {
359 }
360
361 // On z15 we have native support for a 64-bit CTPOP.
362 if (Subtarget.hasMiscellaneousExtensions3()) {
365 }
366
367 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
369
370 // Expand 128 bit shifts without using a libcall.
374
375 // Also expand 256 bit shifts if i128 is a legal type.
376 if (isTypeLegal(MVT::i128)) {
380 }
381
382 // Handle bitcast from fp128 to i128.
383 if (!isTypeLegal(MVT::i128))
385
386 // We have native instructions for i8, i16 and i32 extensions, but not i1.
388 for (MVT VT : MVT::integer_valuetypes()) {
392 }
393
394 // Handle the various types of symbolic address.
400
401 // We need to handle dynamic allocations specially because of the
402 // 160-byte area at the bottom of the stack.
405
408
409 // Handle prefetches with PFD or PFDRL.
411
412 // Handle readcyclecounter with STCKF.
414
416 // Assume by default that all vector operations need to be expanded.
417 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
418 if (getOperationAction(Opcode, VT) == Legal)
419 setOperationAction(Opcode, VT, Expand);
420
421 // Likewise all truncating stores and extending loads.
422 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
423 setTruncStoreAction(VT, InnerVT, Expand);
426 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
427 }
428
429 if (Subtarget.hasVector() && isTypeLegal(VT)) {
430 // These operations are legal for anything that can be stored in a
431 // vector register, even if there is no native support for the format
432 // as such. In particular, we can do these for v4f32 even though there
433 // are no specific instructions for that format.
439
440 // Likewise, except that we need to replace the nodes with something
441 // more specific.
444 }
445 }
446
447 if (!Subtarget.hasVector()) {
448 // A v2i64 GR128 pair can only be built from or split into its halves.
449 // Loads and stores are expanded to avoid tying up a register pair.
452 MVT::v2i64, Legal);
454 MVT::v2i64, Custom);
455 }
456
457 // Handle integer vector types.
459 if (Subtarget.hasVector() && isTypeLegal(VT)) {
460 // These operations have direct equivalents.
465 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
469 }
470 if (Subtarget.hasVectorEnhancements3() &&
471 VT != MVT::v16i8 && VT != MVT::v8i16) {
476 }
481 if (Subtarget.hasVectorEnhancements1())
483 else
487
488 // Convert a GPR scalar to a vector by inserting it into element 0.
490
491 // Use a series of unpacks for extensions.
494
495 // Detect shifts/rotates by a scalar amount and convert them into
496 // V*_BY_SCALAR.
501
502 // Add ISD::VECREDUCE_ADD as custom in order to implement
503 // it with VZERO+VSUM
505
506 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
507 // and inverting the result as necessary.
509
511 Legal);
512 }
513 }
514
515 if (Subtarget.hasVector()) {
516 // There should be no need to check for float types other than v2f64
517 // since <2 x f32> isn't a legal type.
526
535 }
536
537 if (Subtarget.hasVectorEnhancements2()) {
546
555 }
556
557 // Handle floating-point types.
558 if (!useSoftFloat()) {
559 // Promote all f16 operations to float, with some exceptions below.
560 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
561 setOperationAction(Opc, MVT::f16, Promote);
563 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
564 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
565 setTruncStoreAction(VT, MVT::f16, Expand);
566 }
568 setOperationAction(Op, MVT::f16, Subtarget.hasVector() ? Legal : Custom);
572
573 for (auto Op : {ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN})
574 setOperationAction(Op, MVT::f16, Legal);
575 }
576
577 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
578 I <= MVT::LAST_FP_VALUETYPE;
579 ++I) {
581 if (isTypeLegal(VT) && VT != MVT::f16) {
582 // We can use FI for FRINT.
584
585 // We can use the extended form of FI for other rounding operations.
586 if (Subtarget.hasFPExtension()) {
593 }
594
595 // No special instructions for these.
601
602 // Special treatment.
604
605 // Handle constrained floating-point operations.
614 if (Subtarget.hasFPExtension()) {
621 }
622
623 // Extension from f16 needs libcall.
626 }
627 }
628
629 // Handle floating-point vector types.
630 if (Subtarget.hasVector()) {
631 // Scalar-to-vector conversion is just a subreg.
635
636 // Some insertions and extractions can be done directly but others
637 // need to go via integers.
644
645 // These operations have direct equivalents.
646 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
647 setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
648 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
649 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
650 setOperationAction(ISD::FMA, MVT::v2f64, Legal);
651 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
652 setOperationAction(ISD::FABS, MVT::v2f64, Legal);
653 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
654 setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
657 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
661
662 // Handle constrained floating-point operations.
676
681 if (Subtarget.hasVectorEnhancements1()) {
684 }
685 }
686
687 // The vector enhancements facility 1 has instructions for these.
688 if (Subtarget.hasVectorEnhancements1()) {
689 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
690 setOperationAction(ISD::FNEG, MVT::v4f32, Legal);
691 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
692 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
693 setOperationAction(ISD::FMA, MVT::v4f32, Legal);
694 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
695 setOperationAction(ISD::FABS, MVT::v4f32, Legal);
696 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
697 setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
700 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
704
705 for (MVT Type : {MVT::f64, MVT::v2f64, MVT::f32, MVT::v4f32, MVT::f128}) {
714 }
715
716 // Handle constrained floating-point operations.
730 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
731 MVT::v4f32, MVT::v2f64 }) {
738 }
739 }
740
741 // We only have fused f128 multiply-addition on vector registers.
742 if (!Subtarget.hasVectorEnhancements1()) {
745 }
746
747 // We don't have a copysign instruction on vector registers.
748 if (Subtarget.hasVectorEnhancements1())
750
751 // Needed so that we don't try to implement f128 constant loads using
752 // a load-and-extend of a f80 constant (in cases where the constant
753 // would fit in an f80).
754 for (MVT VT : MVT::fp_valuetypes())
755 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
756
757 // We don't have extending load instruction on vector registers.
758 if (Subtarget.hasVectorEnhancements1()) {
759 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
760 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
761 }
762
763 // Floating-point truncation and stores need to be done separately.
764 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
765 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
766 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
767
768 // We have 64-bit FPR<->GPR moves, but need special handling for
769 // 32-bit forms.
770 if (!Subtarget.hasVector()) {
773 }
774
775 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
776 // structure, but VAEND is a no-op.
780
781 if (Subtarget.isTargetzOS()) {
782 // Handle address space casts between mixed sized pointers.
785 }
786
788
789 // Codes for which we want to perform some z-specific combinations.
793 ISD::LOAD,
806 ISD::SRL,
807 ISD::SRA,
808 ISD::MUL,
809 ISD::SDIV,
810 ISD::UDIV,
811 ISD::SREM,
812 ISD::UREM,
815
816 // Handle intrinsics.
819
820 // We're not using SJLJ for exception handling, but they're implemented
821 // solely to support use of __builtin_setjmp / __builtin_longjmp.
824
825 // We want to use MVC in preference to even a single load/store pair.
826 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
828
829 // Same with memmove.
830 MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
832
833 // The main memset sequence is a byte store followed by an MVC.
834 // Two STC or MV..I stores win over that, but the kind of fused stores
835 // generated by target-independent code don't when the byte value is
836 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
837 // than "STC;MVC". Handle the choice in target-specific code instead.
838 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
840
841 // Default to having -disable-strictnode-mutation on
842 IsStrictFPEnabled = true;
843}
844
846 return Subtarget.hasSoftFloat();
847}
848
851 // Widen subvectors to the full width rather than promoting integer
852 // elements. This is better because:
853 //
854 // (a) it means that we can handle the ABI for passing and returning
855 // sub-128 vectors without having to handle them as legal types.
856 //
857 // (b) we don't have instructions to extend on load and truncate on store,
858 // so promoting the integers is less efficient.
859 //
860 // (c) there are no multiplication instructions for the widest integer
861 // type (v2i64).
862
863 // Expand (narrow) f16 vectors during type legalization to avoid
864 // operations for all elements as with expansion after widening. Without
865 // vector support, v2i64 is only legal as a GR128 pair, so do not widen or
866 // promote to it.
867 if (!Subtarget.hasVector() || VT.getScalarType() == MVT::f16)
870 if (VT.getScalarSizeInBits() % 8 == 0)
871 return TypeWidenVector;
873}
874
876 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
877 unsigned &NumIntermediates, MVT &RegisterVT) const {
878 // Pass fp16 vectors in VR(s).
879 if (Subtarget.hasVector() && VT.isVectorOf(MVT::f16)) {
880 IntermediateVT = RegisterVT = MVT::v8f16;
881 return NumIntermediates =
883 }
885 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
886 // Without vector support, v2i64 is a GR128 pair, which is never used to pass
887 // values. Pass the elements individually.
888 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector()) {
889 IntermediateVT = RegisterVT = MVT::i64;
890 NumIntermediates *= 2;
891 NumRegs *= 2;
892 }
893 return NumRegs;
894}
895
898 EVT VT) const {
899 // 128-bit single-element vector types are passed like other vectors,
900 // not like their element type.
901 if (Subtarget.hasVector() && VT.isVector() && VT.getSizeInBits() == 128 &&
902 VT.getVectorNumElements() == 1)
903 return MVT::v16i8;
904 // Pass fp16 vectors in VR(s).
905 if (Subtarget.hasVector() && VT.isVectorOf(MVT::f16))
906 return MVT::v8f16;
907 MVT RegisterVT =
909 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector())
910 return MVT::i64;
911 return RegisterVT;
912}
913
915 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
916 // Pass fp16 vectors in VR(s).
917 if (Subtarget.hasVector() && VT.isVectorOf(MVT::f16))
919 unsigned NumRegs =
921 MVT RegisterVT =
923 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector())
924 return NumRegs * 2;
925 return NumRegs;
926}
927
929 LLVMContext &, EVT VT) const {
930 if (!VT.isVector())
931 return MVT::i32;
933}
934
936 const MachineFunction &MF, EVT VT) const {
937 if (useSoftFloat())
938 return false;
939
940 VT = VT.getScalarType();
941
942 if (!VT.isSimple())
943 return false;
944
945 switch (VT.getSimpleVT().SimpleTy) {
946 case MVT::f32:
947 case MVT::f64:
948 return true;
949 case MVT::f128:
950 return Subtarget.hasVectorEnhancements1();
951 default:
952 break;
953 }
954
955 return false;
956}
957
958// Return true if the constant can be generated with a vector instruction,
959// such as VGM, VGMB or VREPI.
962 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
963 if (!Subtarget.hasVector() ||
964 (isFP128 && !Subtarget.hasVectorEnhancements1()))
965 return false;
966
967 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
968 // preferred way of creating all-zero and all-one vectors so give it
969 // priority over other methods below.
970 unsigned Mask = 0;
971 unsigned I = 0;
972 for (; I < SystemZ::VectorBytes; ++I) {
973 uint64_t Byte = IntBits.lshr(I * 8).trunc(8).getZExtValue();
974 if (Byte == 0xff)
975 Mask |= 1ULL << I;
976 else if (Byte != 0)
977 break;
978 }
979 if (I == SystemZ::VectorBytes) {
980 Opcode = SystemZISD::BYTE_MASK;
981 OpVals.push_back(Mask);
983 return true;
984 }
985
986 if (SplatBitSize > 64)
987 return false;
988
989 auto TryValue = [&](uint64_t Value) -> bool {
990 // Try VECTOR REPLICATE IMMEDIATE
991 int64_t SignedValue = SignExtend64(Value, SplatBitSize);
992 if (isInt<16>(SignedValue)) {
993 OpVals.push_back(((unsigned) SignedValue));
994 Opcode = SystemZISD::REPLICATE;
996 SystemZ::VectorBits / SplatBitSize);
997 return true;
998 }
999 // Try VECTOR GENERATE MASK
1000 unsigned Start, End;
1001 if (TII->isRxSBGMask(Value, SplatBitSize, Start, End)) {
1002 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
1003 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
1004 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
1005 OpVals.push_back(Start - (64 - SplatBitSize));
1006 OpVals.push_back(End - (64 - SplatBitSize));
1007 Opcode = SystemZISD::ROTATE_MASK;
1009 SystemZ::VectorBits / SplatBitSize);
1010 return true;
1011 }
1012 return false;
1013 };
1014
1015 // First try assuming that any undefined bits above the highest set bit
1016 // and below the lowest set bit are 1s. This increases the likelihood of
1017 // being able to use a sign-extended element value in VECTOR REPLICATE
1018 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
1019 uint64_t SplatBitsZ = SplatBits.getZExtValue();
1020 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
1021 unsigned LowerBits = llvm::countr_zero(SplatBitsZ);
1022 unsigned UpperBits = llvm::countl_zero(SplatBitsZ);
1023 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(LowerBits);
1024 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(UpperBits);
1025 if (TryValue(SplatBitsZ | Upper | Lower))
1026 return true;
1027
1028 // Now try assuming that any undefined bits between the first and
1029 // last defined set bits are set. This increases the chances of
1030 // using a non-wraparound mask.
1031 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
1032 return TryValue(SplatBitsZ | Middle);
1033}
1034
1036 if (IntImm.isSingleWord()) {
1037 IntBits = APInt(128, IntImm.getZExtValue());
1038 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
1039 } else
1040 IntBits = IntImm;
1041 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
1042
1043 // Find the smallest splat.
1044 SplatBits = IntImm;
1045 unsigned Width = SplatBits.getBitWidth();
1046 while (Width > 8) {
1047 unsigned HalfSize = Width / 2;
1048 APInt HighValue = SplatBits.lshr(HalfSize).trunc(HalfSize);
1049 APInt LowValue = SplatBits.trunc(HalfSize);
1050
1051 // If the two halves do not match, stop here.
1052 if (HighValue != LowValue || 8 > HalfSize)
1053 break;
1054
1055 SplatBits = HighValue;
1056 Width = HalfSize;
1057 }
1058 SplatUndef = 0;
1059 SplatBitSize = Width;
1060}
1061
1063 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
1064 bool HasAnyUndefs;
1065
1066 // Get IntBits by finding the 128 bit splat.
1067 BVN->isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
1068 true);
1069
1070 // Get SplatBits by finding the 8 bit or greater splat.
1071 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
1072 true);
1073}
1074
1077 bool ForCodeSize) const {
1078 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
1079 if (Imm.isZero() || Imm.isNegZero())
1080 return FPImmAction::Legal;
1081
1085}
1086
1089 MachineBasicBlock *MBB) const {
1090 DebugLoc DL = MI.getDebugLoc();
1091 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1092 const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
1093
1094 MachineFunction *MF = MBB->getParent();
1096
1097 const BasicBlock *BB = MBB->getBasicBlock();
1098 MachineFunction::iterator I = ++MBB->getIterator();
1099
1100 Register DstReg = MI.getOperand(0).getReg();
1101 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
1102 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
1103 (void)TRI;
1104 Register MainDstReg = MRI.createVirtualRegister(RC);
1105 Register RestoreDstReg = MRI.createVirtualRegister(RC);
1106
1107 MVT PVT = getPointerTy(MF->getDataLayout());
1108 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1109 // For v = setjmp(buf), we generate.
1110 // Algorithm:
1111 //
1112 // ---------
1113 // | thisMBB |
1114 // ---------
1115 // |
1116 // ------------------------
1117 // | |
1118 // ---------- ---------------
1119 // | mainMBB | | restoreMBB |
1120 // | v = 0 | | v = 1 |
1121 // ---------- ---------------
1122 // | |
1123 // -------------------------
1124 // |
1125 // -----------------------------
1126 // | sinkMBB |
1127 // | phi(v_mainMBB,v_restoreMBB) |
1128 // -----------------------------
1129 // thisMBB:
1130 // buf[FPOffset] = Frame Pointer if hasFP.
1131 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
1132 // buf[BCOffset] = Backchain value if building with -mbackchain.
1133 // buf[SPOffset] = Stack Pointer.
1134 // buf[LPOffset] = We never write this slot with R13, gcc stores R13 always.
1135 // SjLjSetup restoreMBB
1136 // mainMBB:
1137 // v_main = 0
1138 // sinkMBB:
1139 // v = phi(v_main, v_restore)
1140 // restoreMBB:
1141 // v_restore = 1
1142
1143 MachineBasicBlock *ThisMBB = MBB;
1144 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
1145 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
1146 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
1147
1148 MF->insert(I, MainMBB);
1149 MF->insert(I, SinkMBB);
1150 MF->push_back(RestoreMBB);
1151 RestoreMBB->setMachineBlockAddressTaken();
1152
1154
1155 // Transfer the remainder of BB and its successor edges to sinkMBB.
1156 SinkMBB->splice(SinkMBB->begin(), MBB,
1157 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
1159
1160 // thisMBB:
1161 const int64_t FPOffset = 0; // Slot 1.
1162 const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
1163 const int64_t BCOffset = 2 * PVT.getStoreSize(); // Slot 3.
1164 const int64_t SPOffset = 3 * PVT.getStoreSize(); // Slot 4.
1165
1166 // Buf address.
1167 Register BufReg = MI.getOperand(1).getReg();
1168
1169 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
1170 Register LabelReg = MRI.createVirtualRegister(PtrRC);
1171
1172 // Prepare IP for longjmp.
1173 BuildMI(*ThisMBB, MI, DL, TII->get(SystemZ::LARL), LabelReg)
1174 .addMBB(RestoreMBB);
1175 // Store IP for return from jmp, slot 2, offset = 1.
1176 BuildMI(*ThisMBB, MI, DL, TII->get(SystemZ::STG))
1177 .addReg(LabelReg)
1178 .addReg(BufReg)
1179 .addImm(LabelOffset)
1180 .addReg(0);
1181
1182 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1183 bool HasFP = Subtarget.getFrameLowering()->hasFP(*MF);
1184 if (HasFP) {
1185 BuildMI(*ThisMBB, MI, DL, TII->get(SystemZ::STG))
1186 .addReg(SpecialRegs->getFramePointerRegister())
1187 .addReg(BufReg)
1188 .addImm(FPOffset)
1189 .addReg(0);
1190 }
1191
1192 // Store SP.
1193 BuildMI(*ThisMBB, MI, DL, TII->get(SystemZ::STG))
1194 .addReg(SpecialRegs->getStackPointerRegister())
1195 .addReg(BufReg)
1196 .addImm(SPOffset)
1197 .addReg(0);
1198
1199 // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
1200 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1201 if (BackChain) {
1202 Register BCReg = MRI.createVirtualRegister(PtrRC);
1203 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1204 MIB = BuildMI(*ThisMBB, MI, DL, TII->get(SystemZ::LG), BCReg)
1205 .addReg(SpecialRegs->getStackPointerRegister())
1206 .addImm(TFL->getBackchainOffset(*MF))
1207 .addReg(0);
1208
1209 BuildMI(*ThisMBB, MI, DL, TII->get(SystemZ::STG))
1210 .addReg(BCReg)
1211 .addReg(BufReg)
1212 .addImm(BCOffset)
1213 .addReg(0);
1214 }
1215
1216 // Setup.
1217 MIB = BuildMI(*ThisMBB, MI, DL, TII->get(SystemZ::EH_SjLj_Setup))
1218 .addMBB(RestoreMBB);
1219
1220 const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1221 MIB.addRegMask(RegInfo->getNoPreservedMask());
1222
1223 ThisMBB->addSuccessor(MainMBB);
1224 ThisMBB->addSuccessor(RestoreMBB);
1225
1226 // mainMBB:
1227 BuildMI(MainMBB, DL, TII->get(SystemZ::LHI), MainDstReg).addImm(0);
1228 MainMBB->addSuccessor(SinkMBB);
1229
1230 // sinkMBB:
1231 BuildMI(*SinkMBB, SinkMBB->begin(), DL, TII->get(SystemZ::PHI), DstReg)
1232 .addReg(MainDstReg)
1233 .addMBB(MainMBB)
1234 .addReg(RestoreDstReg)
1235 .addMBB(RestoreMBB);
1236
1237 // restoreMBB.
1238 BuildMI(RestoreMBB, DL, TII->get(SystemZ::LHI), RestoreDstReg).addImm(1);
1239 BuildMI(RestoreMBB, DL, TII->get(SystemZ::J)).addMBB(SinkMBB);
1240 RestoreMBB->addSuccessor(SinkMBB);
1241
1242 MI.eraseFromParent();
1243
1244 return SinkMBB;
1245}
1246
1249 MachineBasicBlock *MBB) const {
1250
1251 DebugLoc DL = MI.getDebugLoc();
1252 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1253
1254 MachineFunction *MF = MBB->getParent();
1256
1257 MVT PVT = getPointerTy(MF->getDataLayout());
1258 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1259 Register BufReg = MI.getOperand(0).getReg();
1260 const TargetRegisterClass *RC = MRI.getRegClass(BufReg);
1261 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1262
1263 Register Tmp = MRI.createVirtualRegister(RC);
1264 Register BCReg = MRI.createVirtualRegister(RC);
1265
1267
1268 const int64_t FPOffset = 0;
1269 const int64_t LabelOffset = 1 * PVT.getStoreSize();
1270 const int64_t BCOffset = 2 * PVT.getStoreSize();
1271 const int64_t SPOffset = 3 * PVT.getStoreSize();
1272 const int64_t LPOffset = 4 * PVT.getStoreSize();
1273
1274 MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG), Tmp)
1275 .addReg(BufReg)
1276 .addImm(LabelOffset)
1277 .addReg(0);
1278
1279 MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG),
1280 SpecialRegs->getFramePointerRegister())
1281 .addReg(BufReg)
1282 .addImm(FPOffset)
1283 .addReg(0);
1284
1285 // We are restoring R13 even though we never stored in setjmp from llvm,
1286 // as gcc always stores R13 in builtin_setjmp. We could have mixed code
1287 // gcc setjmp and llvm longjmp.
1288 MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG), SystemZ::R13D)
1289 .addReg(BufReg)
1290 .addImm(LPOffset)
1291 .addReg(0);
1292
1293 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1294 if (BackChain) {
1295 MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG), BCReg)
1296 .addReg(BufReg)
1297 .addImm(BCOffset)
1298 .addReg(0);
1299 }
1300
1301 MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::LG),
1302 SpecialRegs->getStackPointerRegister())
1303 .addReg(BufReg)
1304 .addImm(SPOffset)
1305 .addReg(0);
1306
1307 if (BackChain) {
1308 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1309 BuildMI(*MBB, MI, DL, TII->get(SystemZ::STG))
1310 .addReg(BCReg)
1311 .addReg(SpecialRegs->getStackPointerRegister())
1312 .addImm(TFL->getBackchainOffset(*MF))
1313 .addReg(0);
1314 }
1315
1316 MIB = BuildMI(*MBB, MI, DL, TII->get(SystemZ::BR)).addReg(Tmp);
1317
1318 MI.eraseFromParent();
1319 return MBB;
1320}
1321
1323 const MachineFunction &MF) const {
1324 // Without vector support, a v2i64 store would just be split again.
1325 return MemVT != MVT::v2i64 || Subtarget.hasVector();
1326}
1327
1329 EVT VecVT) const {
1330 // Without vector support, a v2i64 is just a pair of GPRs.
1331 return !Subtarget.hasVector();
1332}
1333
1334/// Returns true if stack probing through inline assembly is requested.
1336 // If the function specifically requests inline stack probes, emit them.
1337 if (MF.getFunction().hasFnAttribute("probe-stack"))
1338 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
1339 "inline-asm";
1340 return false;
1341}
1342
1347
1352
1355 const AtomicRMWInst *RMW) const {
1356 // Don't expand subword operations as they require special treatment.
1357 if (RMW->getType()->isIntegerTy(8) || RMW->getType()->isIntegerTy(16))
1359
1360 // Don't expand if there is a target instruction available.
1361 if (Subtarget.hasInterlockedAccess1() &&
1362 (RMW->getType()->isIntegerTy(32) || RMW->getType()->isIntegerTy(64)) &&
1369
1371}
1372
1374 // We can use CGFI or CLGFI.
1375 return isInt<32>(Imm) || isUInt<32>(Imm);
1376}
1377
1379 // We can use ALGFI or SLGFI.
1380 return isUInt<32>(Imm) || isUInt<32>(-Imm);
1381}
1382
1384 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
1385 // Unaligned accesses should never be slower than the expanded version.
1386 // We check specifically for aligned accesses in the few cases where
1387 // they are required.
1388 if (Fast)
1389 *Fast = 1;
1390 return true;
1391}
1392
1394 EVT VT = Y.getValueType();
1395
1396 // We can use NC(G)RK for types in GPRs ...
1397 if (VT == MVT::i32 || VT == MVT::i64)
1398 return Subtarget.hasMiscellaneousExtensions3();
1399
1400 // ... or VNC for types in VRs.
1401 if (VT.isVector() || VT == MVT::i128)
1402 return Subtarget.hasVector();
1403
1404 return false;
1405}
1406
1407// Information about the addressing mode for a memory access.
1409 // True if a long displacement is supported.
1411
1412 // True if use of index register is supported.
1414
1415 AddressingMode(bool LongDispl, bool IdxReg) :
1416 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1417};
1418
1419// Return the desired addressing mode for a Load which has only one use (in
1420// the same block) which is a Store.
1422 Type *Ty) {
1423 // With vector support a Load->Store combination may be combined to either
1424 // an MVC or vector operations and it seems to work best to allow the
1425 // vector addressing mode.
1426 if (HasVector)
1427 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1428
1429 // Otherwise only the MVC case is special.
1430 bool MVC = Ty->isIntegerTy(8);
1431 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
1432}
1433
1434// Return the addressing mode which seems most desirable given an LLVM
1435// Instruction pointer.
1436static AddressingMode
1439 switch (II->getIntrinsicID()) {
1440 default: break;
1441 case Intrinsic::memset:
1442 case Intrinsic::memmove:
1443 case Intrinsic::memcpy:
1444 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1445 }
1446 }
1447
1448 if (isa<LoadInst>(I) && I->hasOneUse()) {
1449 auto *SingleUser = cast<Instruction>(*I->user_begin());
1450 if (SingleUser->getParent() == I->getParent()) {
1451 if (isa<ICmpInst>(SingleUser)) {
1452 if (auto *C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
1453 if (C->getBitWidth() <= 64 &&
1454 (isInt<16>(C->getSExtValue()) || isUInt<16>(C->getZExtValue())))
1455 // Comparison of memory with 16 bit signed / unsigned immediate
1456 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1457 } else if (isa<StoreInst>(SingleUser))
1458 // Load->Store
1459 return getLoadStoreAddrMode(HasVector, I->getType());
1460 }
1461 } else if (auto *StoreI = dyn_cast<StoreInst>(I)) {
1462 if (auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
1463 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1464 // Load->Store
1465 return getLoadStoreAddrMode(HasVector, LoadI->getType());
1466 }
1467
1468 if (HasVector && (isa<LoadInst>(I) || isa<StoreInst>(I))) {
1469
1470 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1471 // dependencies (LDE only supports small offsets).
1472 // * Utilize the vector registers to hold floating point
1473 // values (vector load / store instructions only support small
1474 // offsets).
1475
1476 Type *MemAccessTy = (isa<LoadInst>(I) ? I->getType() :
1477 I->getOperand(0)->getType());
1478 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1479 bool IsVectorAccess = MemAccessTy->isVectorTy();
1480
1481 // A store of an extracted vector element will be combined into a VSTE type
1482 // instruction.
1483 if (!IsVectorAccess && isa<StoreInst>(I)) {
1484 Value *DataOp = I->getOperand(0);
1485 if (isa<ExtractElementInst>(DataOp))
1486 IsVectorAccess = true;
1487 }
1488
1489 // A load which gets inserted into a vector element will be combined into a
1490 // VLE type instruction.
1491 if (!IsVectorAccess && isa<LoadInst>(I) && I->hasOneUse()) {
1492 User *LoadUser = *I->user_begin();
1493 if (isa<InsertElementInst>(LoadUser))
1494 IsVectorAccess = true;
1495 }
1496
1497 if (IsFPAccess || IsVectorAccess)
1498 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1499 }
1500
1501 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1502}
1503
1505 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1506 // Punt on globals for now, although they can be used in limited
1507 // RELATIVE LONG cases.
1508 if (AM.BaseGV)
1509 return false;
1510
1511 // Require a 20-bit signed offset.
1512 if (!isInt<20>(AM.BaseOffs))
1513 return false;
1514
1515 bool RequireD12 =
1516 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(128));
1517 AddressingMode SupportedAM(!RequireD12, true);
1518 if (I != nullptr)
1519 SupportedAM = supportedAddressingMode(I, Subtarget.hasVector());
1520
1521 if (!SupportedAM.LongDisplacement && !isUInt<12>(AM.BaseOffs))
1522 return false;
1523
1524 if (!SupportedAM.IndexReg)
1525 // No indexing allowed.
1526 return AM.Scale == 0;
1527 else
1528 // Indexing is OK but no scale factor can be applied.
1529 return AM.Scale == 0 || AM.Scale == 1;
1530}
1531
1533 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
1534 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
1535 const AttributeList &FuncAttributes, EVT *LargestVT) const {
1536
1537 assert(Limit != ~0U &&
1538 "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
1539
1540 if (Op.isZeroMemset())
1541 return false; // Memset zero: Use XC.
1542
1543 const int MVCFastLen = 16;
1544 // Use MVC up to 16 bytes for memcpy. Small memset uses STC/MVI for first
1545 // byte.
1546 if (Op.isMemcpy() && Op.size() <= MVCFastLen)
1547 return false;
1548 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1549 return false;
1550
1551 // Avoid unaligned VL/VST:s.
1552 if ((Op.size() >= 16 && !Op.isAligned(Align(8))) ||
1553 (Op.size() >= 25 && Op.size() <= 31))
1554 return false;
1555
1557 Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
1558}
1559
1561 LLVMContext &Context, const MemOp &Op,
1562 const AttributeList &FuncAttributes) const {
1563 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1564}
1565
1566bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1567 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1568 return false;
1569 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1570 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1571 return FromBits > ToBits;
1572}
1573
1575 if (!FromVT.isInteger() || !ToVT.isInteger())
1576 return false;
1577 unsigned FromBits = FromVT.getFixedSizeInBits();
1578 unsigned ToBits = ToVT.getFixedSizeInBits();
1579 return FromBits > ToBits;
1580}
1581
1583 unsigned Opc = N->getOpcode();
1584 if ((Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP) && VT.isVector() &&
1585 VT.getScalarSizeInBits() < N->getValueType(0).getScalarSizeInBits())
1586 return false;
1588}
1589
1590//===----------------------------------------------------------------------===//
1591// Inline asm support
1592//===----------------------------------------------------------------------===//
1593
1596 if (Constraint.size() == 1) {
1597 switch (Constraint[0]) {
1598 case 'a': // Address register
1599 case 'd': // Data register (equivalent to 'r')
1600 case 'f': // Floating-point register
1601 case 'h': // High-part register
1602 case 'r': // General-purpose register
1603 case 'v': // Vector register
1604 return C_RegisterClass;
1605
1606 case 'Q': // Memory with base and unsigned 12-bit displacement
1607 case 'R': // Likewise, plus an index
1608 case 'S': // Memory with base and signed 20-bit displacement
1609 case 'T': // Likewise, plus an index
1610 case 'm': // Equivalent to 'T'.
1611 return C_Memory;
1612
1613 case 'I': // Unsigned 8-bit constant
1614 case 'J': // Unsigned 12-bit constant
1615 case 'K': // Signed 16-bit constant
1616 case 'L': // Signed 20-bit displacement (on all targets we support)
1617 case 'M': // 0x7fffffff
1618 return C_Immediate;
1619
1620 default:
1621 break;
1622 }
1623 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1624 switch (Constraint[1]) {
1625 case 'Q': // Address with base and unsigned 12-bit displacement
1626 case 'R': // Likewise, plus an index
1627 case 'S': // Address with base and signed 20-bit displacement
1628 case 'T': // Likewise, plus an index
1629 return C_Address;
1630
1631 default:
1632 break;
1633 }
1634 } else if (Constraint.size() == 5 && Constraint.starts_with("{")) {
1635 if (StringRef("{@cc}").compare(Constraint) == 0)
1636 return C_Other;
1637 }
1638 return TargetLowering::getConstraintType(Constraint);
1639}
1640
1643 AsmOperandInfo &Info, const char *Constraint) const {
1645 Value *CallOperandVal = Info.CallOperandVal;
1646 // If we don't have a value, we can't do a match,
1647 // but allow it at the lowest weight.
1648 if (!CallOperandVal)
1649 return CW_Default;
1650 Type *type = CallOperandVal->getType();
1651 // Look at the constraint type.
1652 switch (*Constraint) {
1653 default:
1654 Weight = TargetLowering::getSingleConstraintMatchWeight(Info, Constraint);
1655 break;
1656
1657 case 'a': // Address register
1658 case 'd': // Data register (equivalent to 'r')
1659 case 'h': // High-part register
1660 case 'r': // General-purpose register
1661 Weight =
1662 CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1663 break;
1664
1665 case 'f': // Floating-point register
1666 if (!useSoftFloat())
1667 Weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1668 break;
1669
1670 case 'v': // Vector register
1671 if (Subtarget.hasVector())
1672 Weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1673 : CW_Default;
1674 break;
1675
1676 case 'I': // Unsigned 8-bit constant
1677 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1678 if (isUInt<8>(C->getZExtValue()))
1679 Weight = CW_Constant;
1680 break;
1681
1682 case 'J': // Unsigned 12-bit constant
1683 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1684 if (isUInt<12>(C->getZExtValue()))
1685 Weight = CW_Constant;
1686 break;
1687
1688 case 'K': // Signed 16-bit constant
1689 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1690 if (isInt<16>(C->getSExtValue()))
1691 Weight = CW_Constant;
1692 break;
1693
1694 case 'L': // Signed 20-bit displacement (on all targets we support)
1695 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1696 if (isInt<20>(C->getSExtValue()))
1697 Weight = CW_Constant;
1698 break;
1699
1700 case 'M': // 0x7fffffff
1701 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1702 if (C->getZExtValue() == 0x7fffffff)
1703 Weight = CW_Constant;
1704 break;
1705 }
1706 return Weight;
1707}
1708
1709// Parse a "{tNNN}" register constraint for which the register type "t"
1710// has already been verified. MC is the class associated with "t" and
1711// Map maps 0-based register numbers to LLVM register numbers.
1712static std::pair<unsigned, const TargetRegisterClass *>
1714 const unsigned *Map, unsigned Size) {
1715 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1716 if (isdigit(Constraint[2])) {
1717 unsigned Index;
1718 bool Failed =
1719 Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index);
1720 if (!Failed && Index < Size && Map[Index])
1721 return std::make_pair(Map[Index], RC);
1722 }
1723 return std::make_pair(0U, nullptr);
1724}
1725
1726std::pair<unsigned, const TargetRegisterClass *>
1728 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1729 if (Constraint.size() == 1) {
1730 // GCC Constraint Letters
1731 switch (Constraint[0]) {
1732 default: break;
1733 case 'd': // Data register (equivalent to 'r')
1734 case 'r': // General-purpose register
1735 // MVT::Other has no size, e.g. an aggregate operand that could not be
1736 // converted to an integer type.
1737 if (VT == MVT::Other)
1738 break;
1739 if (VT.getSizeInBits() == 64)
1740 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
1741 else if (VT.getSizeInBits() == 128)
1742 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
1743 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
1744
1745 case 'a': // Address register
1746 if (VT == MVT::i64)
1747 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
1748 else if (VT == MVT::i128)
1749 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
1750 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
1751
1752 case 'h': // High-part register (an LLVM extension)
1753 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
1754
1755 case 'f': // Floating-point register
1756 if (!useSoftFloat()) {
1757 if (VT == MVT::Other)
1758 break;
1759 if (VT.getSizeInBits() == 16)
1760 return std::make_pair(0U, &SystemZ::FP16BitRegClass);
1761 else if (VT.getSizeInBits() == 64)
1762 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
1763 else if (VT.getSizeInBits() == 128)
1764 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
1765 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
1766 }
1767 break;
1768
1769 case 'v': // Vector register
1770 if (Subtarget.hasVector()) {
1771 if (VT == MVT::Other)
1772 break;
1773 if (VT.getSizeInBits() == 16)
1774 return std::make_pair(0U, &SystemZ::VR16BitRegClass);
1775 if (VT.getSizeInBits() == 32)
1776 return std::make_pair(0U, &SystemZ::VR32BitRegClass);
1777 if (VT.getSizeInBits() == 64)
1778 return std::make_pair(0U, &SystemZ::VR64BitRegClass);
1779 return std::make_pair(0U, &SystemZ::VR128BitRegClass);
1780 }
1781 break;
1782 }
1783 }
1784 if (Constraint.starts_with("{")) {
1785
1786 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1787 // to check the size on.
1788 auto getVTSizeInBits = [&VT]() {
1789 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1790 };
1791
1792 // We need to override the default register parsing for GPRs and FPRs
1793 // because the interpretation depends on VT. The internal names of
1794 // the registers are also different from the external names
1795 // (F0D and F0S instead of F0, etc.).
1796 if (Constraint[1] == 'r') {
1797 if (getVTSizeInBits() == 32)
1798 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass,
1800 if (getVTSizeInBits() == 128)
1801 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass,
1803 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass,
1805 }
1806 if (Constraint[1] == 'f') {
1807 if (useSoftFloat())
1808 return std::make_pair(
1809 0u, static_cast<const TargetRegisterClass *>(nullptr));
1810 if (getVTSizeInBits() == 16)
1811 return parseRegisterNumber(Constraint, &SystemZ::FP16BitRegClass,
1813 if (getVTSizeInBits() == 32)
1814 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass,
1816 if (getVTSizeInBits() == 128)
1817 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass,
1819 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass,
1821 }
1822 if (Constraint[1] == 'v') {
1823 if (!Subtarget.hasVector())
1824 return std::make_pair(
1825 0u, static_cast<const TargetRegisterClass *>(nullptr));
1826 if (getVTSizeInBits() == 16)
1827 return parseRegisterNumber(Constraint, &SystemZ::VR16BitRegClass,
1829 if (getVTSizeInBits() == 32)
1830 return parseRegisterNumber(Constraint, &SystemZ::VR32BitRegClass,
1832 if (getVTSizeInBits() == 64)
1833 return parseRegisterNumber(Constraint, &SystemZ::VR64BitRegClass,
1835 return parseRegisterNumber(Constraint, &SystemZ::VR128BitRegClass,
1837 }
1838 if (Constraint[1] == '@') {
1839 if (StringRef("{@cc}").compare(Constraint) == 0)
1840 return std::make_pair(SystemZ::CC, &SystemZ::CCRRegClass);
1841 }
1842 }
1843 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1844}
1845
1846// FIXME? Maybe this could be a TableGen attribute on some registers and
1847// this table could be generated automatically from RegInfo.
1850 const MachineFunction &MF) const {
1851 Register Reg =
1853 .Case("r4", Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1854 : SystemZ::NoRegister)
1855 .Case("r15",
1856 Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1857 .Default(Register());
1858
1859 return Reg;
1860}
1861
1863 ExceptionHandling EH, const Constant *PersonalityFn) const {
1864 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1865}
1866
1868 ExceptionHandling EH, const Constant *PersonalityFn) const {
1869 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1870}
1871
1872// Convert condition code in CCReg to an i32 value.
1874 SDLoc DL(CCReg);
1875 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1876 return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
1877 DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
1878}
1879
1880// Lower @cc targets via setcc.
1882 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
1883 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
1884 if (StringRef("{@cc}").compare(OpInfo.ConstraintCode) != 0)
1885 return SDValue();
1886
1887 // Check that return type is valid.
1888 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
1889 OpInfo.ConstraintVT.getSizeInBits() < 8)
1890 report_fatal_error("Glue output operand is of invalid type");
1891
1892 if (Glue.getNode()) {
1893 Glue = DAG.getCopyFromReg(Chain, DL, SystemZ::CC, MVT::i32, Glue);
1894 Chain = Glue.getValue(1);
1895 } else
1896 Glue = DAG.getCopyFromReg(Chain, DL, SystemZ::CC, MVT::i32);
1897 return getCCResult(DAG, Glue);
1898}
1899
1901 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1902 SelectionDAG &DAG) const {
1903 // Only support length 1 constraints for now.
1904 if (Constraint.size() == 1) {
1905 switch (Constraint[0]) {
1906 case 'I': // Unsigned 8-bit constant
1907 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1908 if (isUInt<8>(C->getZExtValue()))
1909 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1910 Op.getValueType()));
1911 return;
1912
1913 case 'J': // Unsigned 12-bit constant
1914 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1915 if (isUInt<12>(C->getZExtValue()))
1916 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1917 Op.getValueType()));
1918 return;
1919
1920 case 'K': // Signed 16-bit constant
1921 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1922 if (isInt<16>(C->getSExtValue()))
1923 Ops.push_back(DAG.getSignedTargetConstant(
1924 C->getSExtValue(), SDLoc(Op), Op.getValueType()));
1925 return;
1926
1927 case 'L': // Signed 20-bit displacement (on all targets we support)
1928 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1929 if (isInt<20>(C->getSExtValue()))
1930 Ops.push_back(DAG.getSignedTargetConstant(
1931 C->getSExtValue(), SDLoc(Op), Op.getValueType()));
1932 return;
1933
1934 case 'M': // 0x7fffffff
1935 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1936 if (C->getZExtValue() == 0x7fffffff)
1937 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1938 Op.getValueType()));
1939 return;
1940 }
1941 }
1943}
1944
1945//===----------------------------------------------------------------------===//
1946// Calling conventions
1947//===----------------------------------------------------------------------===//
1948
1949#define GET_CALLING_CONV_IMPL
1950#include "SystemZGenCallingConv.inc"
1951
1953 CallingConv::ID) const {
1954 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1955 SystemZ::R14D, 0 };
1956 return ScratchRegs;
1957}
1958
1960 Type *ToType) const {
1961 return isTruncateFree(FromType, ToType);
1962}
1963
1965 return CI->isTailCall();
1966}
1967
1968// Value is a value that has been passed to us in the location described by VA
1969// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1970// any loads onto Chain.
1972 CCValAssign &VA, SDValue Chain,
1973 SDValue Value) {
1974 // If the argument has been promoted from a smaller type, insert an
1975 // assertion to capture this.
1976 if (VA.getLocInfo() == CCValAssign::SExt)
1978 DAG.getValueType(VA.getValVT()));
1979 else if (VA.getLocInfo() == CCValAssign::ZExt)
1981 DAG.getValueType(VA.getValVT()));
1982
1983 if (VA.isExtInLoc())
1984 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value);
1985 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1986 // If this is a short vector argument loaded from the stack,
1987 // extend from i64 to full vector size and then bitcast.
1988 assert(VA.getLocVT() == MVT::i64);
1989 assert(VA.getValVT().isVector());
1990 Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)});
1991 Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
1992 } else
1993 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1994 return Value;
1995}
1996
1997// Value is a value of type VA.getValVT() that we need to copy into
1998// the location described by VA. Return a copy of Value converted to
1999// VA.getValVT(). The caller is responsible for handling indirect values.
2001 CCValAssign &VA, SDValue Value) {
2002 switch (VA.getLocInfo()) {
2003 case CCValAssign::SExt:
2004 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value);
2005 case CCValAssign::ZExt:
2006 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
2007 case CCValAssign::AExt:
2008 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
2009 case CCValAssign::BCvt: {
2010 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
2011 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
2012 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
2013 // For an f32 vararg we need to first promote it to an f64 and then
2014 // bitcast it to an i64.
2015 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
2016 Value = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, Value);
2017 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
2018 ? MVT::v2i64
2019 : VA.getLocVT();
2020 Value = DAG.getNode(ISD::BITCAST, DL, BitCastToType, Value);
2021 // For ELF, this is a short vector argument to be stored to the stack,
2022 // bitcast to v2i64 and then extract first element.
2023 if (BitCastToType == MVT::v2i64)
2024 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
2025 DAG.getConstant(0, DL, MVT::i32));
2026 return Value;
2027 }
2028 case CCValAssign::Full:
2029 return Value;
2030 default:
2031 llvm_unreachable("Unhandled getLocInfo()");
2032 }
2033}
2034
2036 SDLoc DL(In);
2037 SDValue Lo, Hi;
2038 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128)) {
2039 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, In);
2040 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
2041 DAG.getNode(ISD::SRL, DL, MVT::i128, In,
2042 DAG.getConstant(64, DL, MVT::i32)));
2043 } else {
2044 std::tie(Lo, Hi) = DAG.SplitScalar(In, DL, MVT::i64, MVT::i64);
2045 }
2046
2047 return DAG.getNode(SystemZISD::PAIR128, DL, MVT::v2i64, Hi, Lo);
2048}
2049
2051 SDLoc DL(In);
2052 SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
2053 DL, MVT::i64, In);
2054 SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
2055 DL, MVT::i64, In);
2056
2057 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128)) {
2058 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Lo);
2059 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Hi);
2060 Hi = DAG.getNode(ISD::SHL, DL, MVT::i128, Hi,
2061 DAG.getConstant(64, DL, MVT::i32));
2062 return DAG.getNode(ISD::OR, DL, MVT::i128, Lo, Hi);
2063 } else {
2064 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
2065 }
2066}
2067
2069 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
2070 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
2071 EVT ValueVT = Val.getValueType();
2072 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2073 // Inline assembly operand.
2074 Parts[0] = lowerI128ToGR128(DAG, DAG.getBitcast(MVT::i128, Val));
2075 return true;
2076 }
2077
2078 return false;
2079}
2080
2082 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
2083 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
2084 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2085 // Inline assembly operand.
2086 SDValue Res = lowerGR128ToI128(DAG, Parts[0]);
2087 return DAG.getBitcast(ValueVT, Res);
2088 }
2089
2090 return SDValue();
2091}
2092
2093// The first part of a split stack argument is at index I in Args (and
2094// ArgLocs). Return the type of a part and the number of them by reference.
2095template <class ArgTy>
2097 SmallVector<CCValAssign, 16> &ArgLocs, unsigned I,
2098 MVT &PartVT, unsigned &NumParts) {
2099 if (!Args[I].Flags.isSplit())
2100 return false;
2101 assert(I < ArgLocs.size() && ArgLocs.size() == Args.size() &&
2102 "ArgLocs havoc.");
2103 PartVT = ArgLocs[I].getValVT();
2104 NumParts = 1;
2105 for (unsigned PartIdx = I + 1;; ++PartIdx) {
2106 assert(PartIdx != ArgLocs.size() && "SplitEnd not found.");
2107 assert(ArgLocs[PartIdx].getValVT() == PartVT && "Unsupported split.");
2108 ++NumParts;
2109 if (Args[PartIdx].Flags.isSplitEnd())
2110 break;
2111 }
2112 return true;
2113}
2114
2115// Return the virtual register holding the incoming "ADA" special register (R5)
2116// of an XPLINK function, creating it on first use. LowerFormalArguments is not
2117// called for naked functions, but those still get R5 on entry and may use it to
2118// address globals and functions.
2120 SystemZMachineFunctionInfo *FuncInfo =
2122 Register ADAvReg = FuncInfo->getADAVirtualRegister();
2123 if (ADAvReg)
2124 return ADAvReg;
2125
2127 ADAvReg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
2128 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2130 MRI.addLiveIn(Regs->getADARegister(), ADAvReg);
2131 FuncInfo->setADAVirtualRegister(ADAvReg);
2132 return ADAvReg;
2133}
2134
2136 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
2137 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2138 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2140 MachineFrameInfo &MFI = MF.getFrameInfo();
2142 SystemZMachineFunctionInfo *FuncInfo =
2144 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
2145 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2146
2147 // Assign locations to all of the incoming arguments.
2149 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2150 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ);
2151 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
2152
2153 unsigned NumFixedGPRs = 0;
2154 unsigned NumFixedFPRs = 0;
2155 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2156 SDValue ArgValue;
2157 CCValAssign &VA = ArgLocs[I];
2158 EVT LocVT = VA.getLocVT();
2159 if (VA.isRegLoc()) {
2160 // Arguments passed in registers
2161 const TargetRegisterClass *RC;
2162 switch (LocVT.getSimpleVT().SimpleTy) {
2163 default:
2164 // Integers smaller than i64 should be promoted to i64.
2165 llvm_unreachable("Unexpected argument type");
2166 case MVT::i32:
2167 NumFixedGPRs += 1;
2168 RC = &SystemZ::GR32BitRegClass;
2169 break;
2170 case MVT::i64:
2171 NumFixedGPRs += 1;
2172 RC = &SystemZ::GR64BitRegClass;
2173 break;
2174 case MVT::f16:
2175 NumFixedFPRs += 1;
2176 RC = &SystemZ::FP16BitRegClass;
2177 break;
2178 case MVT::f32:
2179 NumFixedFPRs += 1;
2180 RC = &SystemZ::FP32BitRegClass;
2181 break;
2182 case MVT::f64:
2183 NumFixedFPRs += 1;
2184 RC = &SystemZ::FP64BitRegClass;
2185 break;
2186 case MVT::f128:
2187 NumFixedFPRs += 2;
2188 RC = &SystemZ::FP128BitRegClass;
2189 break;
2190 case MVT::v16i8:
2191 case MVT::v8i16:
2192 case MVT::v4i32:
2193 case MVT::v2i64:
2194 case MVT::v8f16:
2195 case MVT::v4f32:
2196 case MVT::v2f64:
2197 RC = &SystemZ::VR128BitRegClass;
2198 break;
2199 }
2200
2201 Register VReg = MRI.createVirtualRegister(RC);
2202 MRI.addLiveIn(VA.getLocReg(), VReg);
2203 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
2204 } else {
2205 assert(VA.isMemLoc() && "Argument not register or memory");
2206
2207 // Create the frame index object for this incoming parameter.
2208 // FIXME: Pre-include call frame size in the offset, should not
2209 // need to manually add it here.
2210 int64_t ArgSPOffset = VA.getLocMemOffset();
2211 if (Subtarget.isTargetXPLINK64()) {
2212 auto &XPRegs =
2213 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
2214 ArgSPOffset += XPRegs.getCallFrameSize();
2215 }
2216 int FI =
2217 MFI.CreateFixedObject(LocVT.getSizeInBits() / 8, ArgSPOffset, true);
2218
2219 // Create the SelectionDAG nodes corresponding to a load
2220 // from this parameter. Unpromoted ints and floats are
2221 // passed as right-justified 8-byte values.
2222 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
2223 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
2224 VA.getLocVT() == MVT::f16) {
2225 unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
2226 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
2227 DAG.getIntPtrConstant(SlotOffs, DL));
2228 }
2229 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
2231 }
2232
2233 // Convert the value of the argument register into the value that's
2234 // being passed.
2235 if (VA.getLocInfo() == CCValAssign::Indirect) {
2236 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
2238 // If the original argument was split (e.g. i128), we need
2239 // to load all parts of it here (using the same address).
2240 MVT PartVT;
2241 unsigned NumParts;
2242 if (analyzeArgSplit(Ins, ArgLocs, I, PartVT, NumParts)) {
2243 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2244 ++I;
2245 CCValAssign &PartVA = ArgLocs[I];
2246 unsigned PartOffset = Ins[I].PartOffset;
2247 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
2248 DAG.getIntPtrConstant(PartOffset, DL));
2249 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
2251 assert(PartOffset && "Offset should be non-zero.");
2252 }
2253 }
2254 } else if (Subtarget.isTargetXPLINK64() &&
2255 (VA.getLocInfo() == CCValAssign::SExt ||
2256 VA.getLocInfo() == CCValAssign::ZExt) &&
2257 Ins[I].ArgVT.isSimple()) {
2258 // Some prior z/OS compilers do not always perform the extension of
2259 // short integer arguments or pointers. To accommodate those, do not
2260 // rely on that extension by avoiding any AssertSext/AssertZext nodes by
2261 // directly truncating ArgValue to the original argument type.
2262 MVT OrigVT = Ins[I].ArgVT.getSimpleVT();
2263 InVals.push_back(DAG.getNode(ISD::TRUNCATE, DL, OrigVT, ArgValue));
2264 } else
2265 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
2266 }
2267
2268 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2269 // Save the number of non-varargs registers for later use by va_start, etc.
2270 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2271 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2272
2273 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2274 Subtarget.getSpecialRegisters());
2275
2276 // Likewise the address (in the form of a frame index) of where the
2277 // first stack vararg would be. The 1-byte size here is arbitrary.
2278 // FIXME: Pre-include call frame size in the offset, should not
2279 // need to manually add it here.
2280 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
2281 int FI = MFI.CreateFixedObject(1, VarArgOffset, true);
2282 FuncInfo->setVarArgsFrameIndex(FI);
2283 }
2284
2285 if (IsVarArg && Subtarget.isTargetELF()) {
2286 // Save the number of non-varargs registers for later use by va_start, etc.
2287 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2288 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2289
2290 // Likewise the address (in the form of a frame index) of where the
2291 // first stack vararg would be. The 1-byte size here is arbitrary.
2292 int64_t VarArgsOffset = CCInfo.getStackSize();
2293 FuncInfo->setVarArgsFrameIndex(
2294 MFI.CreateFixedObject(1, VarArgsOffset, true));
2295
2296 // ...and a similar frame index for the caller-allocated save area
2297 // that will be used to store the incoming registers.
2298 int64_t RegSaveOffset =
2299 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, SystemZ::R2D) - 16;
2300 unsigned RegSaveIndex = MFI.CreateFixedObject(1, RegSaveOffset, true);
2301 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
2302
2303 // Store the FPR varargs in the reserved frame slots. (We store the
2304 // GPRs as part of the prologue.)
2305 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
2307 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
2308 unsigned Offset = TFL->getRegSpillOffset(MF, SystemZ::ELFArgFPRs[I]);
2309 int FI =
2311 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2313 &SystemZ::FP64BitRegClass);
2314 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
2315 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
2317 }
2318 // Join the stores, which are independent of one another.
2319 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2320 ArrayRef(&MemOps[NumFixedFPRs],
2321 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
2322 }
2323 }
2324
2325 return Chain;
2326}
2327
2328static bool canUseSiblingCall(const CCState &ArgCCInfo,
2331 // Punt if there are any indirect or stack arguments, or if the call
2332 // needs the callee-saved argument register R6, or if the call uses
2333 // the callee-saved register arguments SwiftSelf and SwiftError.
2334 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2335 CCValAssign &VA = ArgLocs[I];
2337 return false;
2338 if (!VA.isRegLoc())
2339 return false;
2340 Register Reg = VA.getLocReg();
2341 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
2342 return false;
2343 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
2344 return false;
2345 }
2346 return true;
2347}
2348
2350 unsigned Offset, bool LoadAdr = false) {
2354
2355 SDValue Reg = DAG.getRegister(ADAvReg, PtrVT);
2356 SDValue Ofs = DAG.getTargetConstant(Offset, DL, PtrVT);
2357
2358 SDValue Result = DAG.getNode(SystemZISD::ADA_ENTRY, DL, PtrVT, Val, Reg, Ofs);
2359 if (!LoadAdr)
2360 Result = DAG.getLoad(
2361 PtrVT, DL, DAG.getEntryNode(), Result, MachinePointerInfo(), Align(8),
2363
2364 return Result;
2365}
2366
2367// ADA access using Global value
2368// Note: for functions, address of descriptor is returned
2370 EVT PtrVT) {
2371 unsigned ADAtype;
2372 bool LoadAddr = false;
2373 const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV);
2374 bool IsFunction =
2375 (isa<Function>(GV)) || (GA && isa<Function>(GA->getAliaseeObject()));
2376 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
2377
2378 if (IsFunction) {
2379 if (IsInternal) {
2381 LoadAddr = true;
2382 } else
2384 } else {
2386 }
2387 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, ADAtype);
2388
2389 return getADAEntry(DAG, Val, DL, 0, LoadAddr);
2390}
2391
2392static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2393 SDLoc &DL, SDValue &Chain) {
2394 unsigned ADADelta = 0; // ADA offset in desc.
2395 unsigned EPADelta = 8; // EPA offset in desc.
2398
2399 // XPLink calling convention.
2400 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2401 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
2402 G->getGlobal()->hasPrivateLinkage());
2403 if (IsInternal) {
2405 ADA = DAG.getCopyFromReg(Chain, DL, ADAvReg, PtrVT);
2406 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
2407 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2408 return true;
2409 } else {
2411 G->getGlobal(), DL, PtrVT, 0, SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2412 ADA = getADAEntry(DAG, GA, DL, ADADelta);
2413 Callee = getADAEntry(DAG, GA, DL, EPADelta);
2414 }
2415 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2417 E->getSymbol(), PtrVT, SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2418 ADA = getADAEntry(DAG, ES, DL, ADADelta);
2419 Callee = getADAEntry(DAG, ES, DL, EPADelta);
2420 } else {
2421 // Function pointer case
2422 ADA = DAG.getNode(ISD::ADD, DL, PtrVT, Callee,
2423 DAG.getConstant(ADADelta, DL, PtrVT));
2424 ADA = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), ADA,
2426 Callee = DAG.getNode(ISD::ADD, DL, PtrVT, Callee,
2427 DAG.getConstant(EPADelta, DL, PtrVT));
2428 Callee = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Callee,
2430 }
2431 return false;
2432}
2433
2434SDValue
2436 SmallVectorImpl<SDValue> &InVals) const {
2437 SelectionDAG &DAG = CLI.DAG;
2438 SDLoc &DL = CLI.DL;
2440 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2442 SDValue Chain = CLI.Chain;
2443 SDValue Callee = CLI.Callee;
2444 bool &IsTailCall = CLI.IsTailCall;
2445 CallingConv::ID CallConv = CLI.CallConv;
2446 bool IsVarArg = CLI.IsVarArg;
2448 EVT PtrVT = getPointerTy(MF.getDataLayout());
2449 LLVMContext &Ctx = *DAG.getContext();
2450 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
2451
2452 // FIXME: z/OS support to be added in later.
2453 if (Subtarget.isTargetXPLINK64()) {
2454 if (CLI.CB && CLI.CB->isMustTailCall())
2455 report_fatal_error("musttail call is not supported on XPLINK64");
2456 IsTailCall = false;
2457 }
2458
2459 // Integer args <=32 bits should have an extension attribute.
2460 verifyNarrowIntegerArgs_Call(CLI, Outs, &MF.getFunction(), Callee);
2461
2462 // Analyze the operands of the call, assigning locations to each operand.
2464 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2465 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
2466
2467 // We don't support GuaranteedTailCallOpt, only automatically-detected
2468 // sibling calls.
2469 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
2470 IsTailCall = false;
2471
2472 if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
2473 report_fatal_error("failed to perform tail call elimination on a call "
2474 "site marked musttail");
2475
2476 // Get a count of how many bytes are to be pushed on the stack.
2477 unsigned NumBytes = ArgCCInfo.getStackSize();
2478
2479 // Mark the start of the call.
2480 if (!IsTailCall)
2481 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
2482
2483 // Copy argument values to their designated locations.
2485 SmallVector<SDValue, 8> MemOpChains;
2486 SDValue StackPtr;
2487 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2488 CCValAssign &VA = ArgLocs[I];
2489 SDValue ArgValue = OutVals[I];
2490
2491 if (VA.getLocInfo() == CCValAssign::Indirect) {
2492 // Store the argument in a stack slot and pass its address.
2493 EVT SlotVT;
2494 MVT PartVT;
2495 unsigned NumParts = 1;
2496 if (analyzeArgSplit(Outs, ArgLocs, I, PartVT, NumParts))
2497 SlotVT = EVT::getIntegerVT(Ctx, PartVT.getSizeInBits() * NumParts);
2498 else
2499 SlotVT = Outs[I].VT;
2500 SDValue SpillSlot = DAG.CreateStackTemporary(SlotVT);
2501 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
2502
2503 MachinePointerInfo StackPtrInfo =
2505 MemOpChains.push_back(
2506 DAG.getStore(Chain, DL, ArgValue, SpillSlot, StackPtrInfo));
2507 // If the original argument was split (e.g. i128), we need
2508 // to store all parts of it here (and pass just one address).
2509 assert(Outs[I].PartOffset == 0);
2510 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2511 ++I;
2512 SDValue PartValue = OutVals[I];
2513 unsigned PartOffset = Outs[I].PartOffset;
2514 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
2515 DAG.getIntPtrConstant(PartOffset, DL));
2516 MemOpChains.push_back(
2517 DAG.getStore(Chain, DL, PartValue, Address,
2518 StackPtrInfo.getWithOffset(PartOffset)));
2519 assert(PartOffset && "Offset should be non-zero.");
2520 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
2521 SlotVT.getStoreSize()) && "Not enough space for argument part!");
2522 }
2523 ArgValue = SpillSlot;
2524 } else
2525 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue);
2526
2527 if (VA.isRegLoc()) {
2528 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
2529 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
2530 // and low values.
2531 if (VA.getLocVT() == MVT::i128)
2532 ArgValue = lowerI128ToGR128(DAG, ArgValue);
2533 // Queue up the argument copies and emit them at the end.
2534 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
2535 } else {
2536 assert(VA.isMemLoc() && "Argument not register or memory");
2537
2538 // Work out the address of the stack slot. Unpromoted ints and
2539 // floats are passed as right-justified 8-byte values.
2540 if (!StackPtr.getNode())
2541 StackPtr = DAG.getCopyFromReg(Chain, DL,
2542 Regs->getStackPointerRegister(), PtrVT);
2543 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
2544 VA.getLocMemOffset();
2545 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
2546 Offset += 4;
2547 else if (VA.getLocVT() == MVT::f16)
2548 Offset += 6;
2549 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
2551
2552 // Emit the store.
2553 MemOpChains.push_back(
2554 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
2555
2556 // Although long doubles or vectors are passed through the stack when
2557 // they are vararg (non-fixed arguments), if a long double or vector
2558 // occupies the third and fourth slot of the argument list GPR3 should
2559 // still shadow the third slot of the argument list.
2560 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
2561 SDValue ShadowArgValue =
2562 DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, ArgValue,
2563 DAG.getIntPtrConstant(1, DL));
2564 RegsToPass.push_back(std::make_pair(SystemZ::R3D, ShadowArgValue));
2565 }
2566 }
2567 }
2568
2569 // Join the stores, which are independent of one another.
2570 if (!MemOpChains.empty())
2571 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2572
2573 // Accept direct calls by converting symbolic call addresses to the
2574 // associated Target* opcodes. Force %r1 to be used for indirect
2575 // tail calls.
2576 SDValue Glue;
2577
2578 if (Subtarget.isTargetXPLINK64()) {
2579 SDValue ADA;
2580 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2581 if (!IsBRASL) {
2582 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2583 ->getAddressOfCalleeRegister();
2584 Chain = DAG.getCopyToReg(Chain, DL, CalleeReg, Callee, Glue);
2585 Glue = Chain.getValue(1);
2586 Callee = DAG.getRegister(CalleeReg, Callee.getValueType());
2587 }
2588 RegsToPass.push_back(std::make_pair(
2589 static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), ADA));
2590 } else {
2591 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2592 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
2593 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2594 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2595 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT);
2596 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2597 } else if (IsTailCall) {
2598 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
2599 Glue = Chain.getValue(1);
2600 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType());
2601 }
2602 }
2603
2604 // Build a sequence of copy-to-reg nodes, chained and glued together.
2605 for (const auto &[Reg, N] : RegsToPass) {
2606 Chain = DAG.getCopyToReg(Chain, DL, Reg, N, Glue);
2607 Glue = Chain.getValue(1);
2608 }
2609
2610 // The first call operand is the chain and the second is the target address.
2612 Ops.push_back(Chain);
2613 Ops.push_back(Callee);
2614
2615 // Add argument registers to the end of the list so that they are
2616 // known live into the call.
2617 for (const auto &[Reg, N] : RegsToPass)
2618 Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
2619
2620 // Add a register mask operand representing the call-preserved registers.
2621 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2622 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2623 assert(Mask && "Missing call preserved mask for calling convention");
2624 Ops.push_back(DAG.getRegisterMask(Mask));
2625
2626 // Glue the call to the argument copies, if any.
2627 if (Glue.getNode())
2628 Ops.push_back(Glue);
2629
2630 // Emit the call.
2631 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2632 if (IsTailCall) {
2633 SDValue Ret = DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
2634 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
2635 return Ret;
2636 }
2637 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
2638 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
2639 Glue = Chain.getValue(1);
2640
2641 // Mark the end of the call, which is glued to the call itself.
2642 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL);
2643 Glue = Chain.getValue(1);
2644
2645 // Assign locations to each value returned by this call.
2647 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2648 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ);
2649
2650 // Copy all of the result registers out of their specified physreg.
2651 for (CCValAssign &VA : RetLocs) {
2652 // Copy the value out, gluing the copy to the end of the call sequence.
2653 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
2654 VA.getLocVT(), Glue);
2655 Chain = RetValue.getValue(1);
2656 Glue = RetValue.getValue(2);
2657
2658 // Convert the value of the return register into the value that's
2659 // being returned.
2660 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue));
2661 }
2662
2663 return Chain;
2664}
2665
2666// Generate a call taking the given operands as arguments and returning a
2667// result of type RetVT.
2669 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2670 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2671 bool DoesNotReturn, bool IsReturnValueUsed) const {
2673 Args.reserve(Ops.size());
2674
2675 for (SDValue Op : Ops) {
2677 Op, Op.getValueType().getTypeForEVT(*DAG.getContext()));
2678 Entry.IsSExt = shouldSignExtendTypeInLibCall(Entry.Ty, IsSigned);
2679 Entry.IsZExt = !Entry.IsSExt;
2680 Args.push_back(Entry);
2681 }
2682
2683 SDValue Callee =
2684 DAG.getExternalSymbol(CalleeName, getPointerTy(DAG.getDataLayout()));
2685
2686 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2688 bool SignExtend = shouldSignExtendTypeInLibCall(RetTy, IsSigned);
2689 CLI.setDebugLoc(DL)
2690 .setChain(Chain)
2691 .setCallee(CallConv, RetTy, Callee, std::move(Args))
2692 .setNoReturn(DoesNotReturn)
2693 .setDiscardResult(!IsReturnValueUsed)
2694 .setSExtResult(SignExtend)
2695 .setZExtResult(!SignExtend);
2696 return LowerCallTo(CLI);
2697}
2698
2700 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2701 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
2702 const Type *RetTy) const {
2703 // Special case that we cannot easily detect in RetCC_SystemZ since
2704 // i128 may not be a legal type.
2705 for (auto &Out : Outs)
2706 if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64)
2707 return false;
2708
2710 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2711 return RetCCInfo.CheckReturn(Outs, RetCC_SystemZ);
2712}
2713
2714SDValue
2716 bool IsVarArg,
2718 const SmallVectorImpl<SDValue> &OutVals,
2719 const SDLoc &DL, SelectionDAG &DAG) const {
2721
2722 // Integer args <=32 bits should have an extension attribute.
2723 verifyNarrowIntegerArgs_Ret(CallConv, Outs, &MF.getFunction());
2724
2725 // Assign locations to each returned value.
2727 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2728 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ);
2729
2730 // Quick exit for void returns
2731 if (RetLocs.empty())
2732 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, Chain);
2733
2734 if (CallConv == CallingConv::GHC)
2735 report_fatal_error("GHC functions return void only");
2736
2737 // Copy the result values into the output registers.
2738 SDValue Glue;
2740 RetOps.push_back(Chain);
2741 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2742 CCValAssign &VA = RetLocs[I];
2743 SDValue RetValue = OutVals[I];
2744
2745 // Make the return register live on exit.
2746 assert(VA.isRegLoc() && "Can only return in registers!");
2747
2748 // Promote the value as required.
2749 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue);
2750
2751 // Chain and glue the copies together.
2752 Register Reg = VA.getLocReg();
2753 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue);
2754 Glue = Chain.getValue(1);
2755 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT()));
2756 }
2757
2758 // Update chain and glue.
2759 RetOps[0] = Chain;
2760 if (Glue.getNode())
2761 RetOps.push_back(Glue);
2762
2763 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, RetOps);
2764}
2765
2766// Return true if Op is an intrinsic node with chain that returns the CC value
2767// as its only (other) argument. Provide the associated SystemZISD opcode and
2768// the mask of valid CC values if so.
2769static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2770 unsigned &CCValid) {
2771 unsigned Id = Op.getConstantOperandVal(1);
2772 switch (Id) {
2773 case Intrinsic::s390_tbegin:
2774 Opcode = SystemZISD::TBEGIN;
2775 CCValid = SystemZ::CCMASK_TBEGIN;
2776 return true;
2777
2778 case Intrinsic::s390_tbegin_nofloat:
2779 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2780 CCValid = SystemZ::CCMASK_TBEGIN;
2781 return true;
2782
2783 case Intrinsic::s390_tend:
2784 Opcode = SystemZISD::TEND;
2785 CCValid = SystemZ::CCMASK_TEND;
2786 return true;
2787
2788 default:
2789 return false;
2790 }
2791}
2792
2793// Return true if Op is an intrinsic node without chain that returns the
2794// CC value as its final argument. Provide the associated SystemZISD
2795// opcode and the mask of valid CC values if so.
2796static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2797 unsigned Id = Op.getConstantOperandVal(0);
2798 switch (Id) {
2799 case Intrinsic::s390_vpkshs:
2800 case Intrinsic::s390_vpksfs:
2801 case Intrinsic::s390_vpksgs:
2802 Opcode = SystemZISD::PACKS_CC;
2803 CCValid = SystemZ::CCMASK_VCMP;
2804 return true;
2805
2806 case Intrinsic::s390_vpklshs:
2807 case Intrinsic::s390_vpklsfs:
2808 case Intrinsic::s390_vpklsgs:
2809 Opcode = SystemZISD::PACKLS_CC;
2810 CCValid = SystemZ::CCMASK_VCMP;
2811 return true;
2812
2813 case Intrinsic::s390_vceqbs:
2814 case Intrinsic::s390_vceqhs:
2815 case Intrinsic::s390_vceqfs:
2816 case Intrinsic::s390_vceqgs:
2817 case Intrinsic::s390_vceqqs:
2818 Opcode = SystemZISD::VICMPES;
2819 CCValid = SystemZ::CCMASK_VCMP;
2820 return true;
2821
2822 case Intrinsic::s390_vchbs:
2823 case Intrinsic::s390_vchhs:
2824 case Intrinsic::s390_vchfs:
2825 case Intrinsic::s390_vchgs:
2826 case Intrinsic::s390_vchqs:
2827 Opcode = SystemZISD::VICMPHS;
2828 CCValid = SystemZ::CCMASK_VCMP;
2829 return true;
2830
2831 case Intrinsic::s390_vchlbs:
2832 case Intrinsic::s390_vchlhs:
2833 case Intrinsic::s390_vchlfs:
2834 case Intrinsic::s390_vchlgs:
2835 case Intrinsic::s390_vchlqs:
2836 Opcode = SystemZISD::VICMPHLS;
2837 CCValid = SystemZ::CCMASK_VCMP;
2838 return true;
2839
2840 case Intrinsic::s390_vtm:
2841 Opcode = SystemZISD::VTM;
2842 CCValid = SystemZ::CCMASK_VCMP;
2843 return true;
2844
2845 case Intrinsic::s390_vfaebs:
2846 case Intrinsic::s390_vfaehs:
2847 case Intrinsic::s390_vfaefs:
2848 Opcode = SystemZISD::VFAE_CC;
2849 CCValid = SystemZ::CCMASK_ANY;
2850 return true;
2851
2852 case Intrinsic::s390_vfaezbs:
2853 case Intrinsic::s390_vfaezhs:
2854 case Intrinsic::s390_vfaezfs:
2855 Opcode = SystemZISD::VFAEZ_CC;
2856 CCValid = SystemZ::CCMASK_ANY;
2857 return true;
2858
2859 case Intrinsic::s390_vfeebs:
2860 case Intrinsic::s390_vfeehs:
2861 case Intrinsic::s390_vfeefs:
2862 Opcode = SystemZISD::VFEE_CC;
2863 CCValid = SystemZ::CCMASK_ANY;
2864 return true;
2865
2866 case Intrinsic::s390_vfeezbs:
2867 case Intrinsic::s390_vfeezhs:
2868 case Intrinsic::s390_vfeezfs:
2869 Opcode = SystemZISD::VFEEZ_CC;
2870 CCValid = SystemZ::CCMASK_ANY;
2871 return true;
2872
2873 case Intrinsic::s390_vfenebs:
2874 case Intrinsic::s390_vfenehs:
2875 case Intrinsic::s390_vfenefs:
2876 Opcode = SystemZISD::VFENE_CC;
2877 CCValid = SystemZ::CCMASK_ANY;
2878 return true;
2879
2880 case Intrinsic::s390_vfenezbs:
2881 case Intrinsic::s390_vfenezhs:
2882 case Intrinsic::s390_vfenezfs:
2883 Opcode = SystemZISD::VFENEZ_CC;
2884 CCValid = SystemZ::CCMASK_ANY;
2885 return true;
2886
2887 case Intrinsic::s390_vistrbs:
2888 case Intrinsic::s390_vistrhs:
2889 case Intrinsic::s390_vistrfs:
2890 Opcode = SystemZISD::VISTR_CC;
2892 return true;
2893
2894 case Intrinsic::s390_vstrcbs:
2895 case Intrinsic::s390_vstrchs:
2896 case Intrinsic::s390_vstrcfs:
2897 Opcode = SystemZISD::VSTRC_CC;
2898 CCValid = SystemZ::CCMASK_ANY;
2899 return true;
2900
2901 case Intrinsic::s390_vstrczbs:
2902 case Intrinsic::s390_vstrczhs:
2903 case Intrinsic::s390_vstrczfs:
2904 Opcode = SystemZISD::VSTRCZ_CC;
2905 CCValid = SystemZ::CCMASK_ANY;
2906 return true;
2907
2908 case Intrinsic::s390_vstrsb:
2909 case Intrinsic::s390_vstrsh:
2910 case Intrinsic::s390_vstrsf:
2911 Opcode = SystemZISD::VSTRS_CC;
2912 CCValid = SystemZ::CCMASK_ANY;
2913 return true;
2914
2915 case Intrinsic::s390_vstrszb:
2916 case Intrinsic::s390_vstrszh:
2917 case Intrinsic::s390_vstrszf:
2918 Opcode = SystemZISD::VSTRSZ_CC;
2919 CCValid = SystemZ::CCMASK_ANY;
2920 return true;
2921
2922 case Intrinsic::s390_vfcedbs:
2923 case Intrinsic::s390_vfcesbs:
2924 Opcode = SystemZISD::VFCMPES;
2925 CCValid = SystemZ::CCMASK_VCMP;
2926 return true;
2927
2928 case Intrinsic::s390_vfchdbs:
2929 case Intrinsic::s390_vfchsbs:
2930 Opcode = SystemZISD::VFCMPHS;
2931 CCValid = SystemZ::CCMASK_VCMP;
2932 return true;
2933
2934 case Intrinsic::s390_vfchedbs:
2935 case Intrinsic::s390_vfchesbs:
2936 Opcode = SystemZISD::VFCMPHES;
2937 CCValid = SystemZ::CCMASK_VCMP;
2938 return true;
2939
2940 case Intrinsic::s390_vftcidb:
2941 case Intrinsic::s390_vftcisb:
2942 Opcode = SystemZISD::VFTCI;
2943 CCValid = SystemZ::CCMASK_VCMP;
2944 return true;
2945
2946 case Intrinsic::s390_tdc:
2947 Opcode = SystemZISD::TDC;
2948 CCValid = SystemZ::CCMASK_TDC;
2949 return true;
2950
2951 default:
2952 return false;
2953 }
2954}
2955
2956// Emit an intrinsic with chain and an explicit CC register result.
2958 unsigned Opcode) {
2959 // Copy all operands except the intrinsic ID.
2960 unsigned NumOps = Op.getNumOperands();
2962 Ops.reserve(NumOps - 1);
2963 Ops.push_back(Op.getOperand(0));
2964 for (unsigned I = 2; I < NumOps; ++I)
2965 Ops.push_back(Op.getOperand(I));
2966
2967 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2968 SDVTList RawVTs = DAG.getVTList(MVT::i32, MVT::Other);
2969 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
2970 SDValue OldChain = SDValue(Op.getNode(), 1);
2971 SDValue NewChain = SDValue(Intr.getNode(), 1);
2972 DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain);
2973 return Intr.getNode();
2974}
2975
2976// Emit an intrinsic with an explicit CC register result.
2978 unsigned Opcode) {
2979 // Copy all operands except the intrinsic ID.
2980 SDLoc DL(Op);
2981 unsigned NumOps = Op.getNumOperands();
2983 Ops.reserve(NumOps - 1);
2984 for (unsigned I = 1; I < NumOps; ++I) {
2985 SDValue CurrOper = Op.getOperand(I);
2986 if (CurrOper.getValueType() == MVT::f16) {
2987 assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
2988 "Unhandled intrinsic with f16 operand.");
2989 CurrOper = DAG.getFPExtendOrRound(CurrOper, DL, MVT::f32);
2990 }
2991 Ops.push_back(CurrOper);
2992 }
2993
2994 SDValue Intr = DAG.getNode(Opcode, DL, Op->getVTList(), Ops);
2995 return Intr.getNode();
2996}
2997
2998// CC is a comparison that will be implemented using an integer or
2999// floating-point comparison. Return the condition code mask for
3000// a branch on true. In the integer case, CCMASK_CMP_UO is set for
3001// unsigned comparisons and clear for signed ones. In the floating-point
3002// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
3004#define CONV(X) \
3005 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
3006 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
3007 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
3008
3009 switch (CC) {
3010 default:
3011 llvm_unreachable("Invalid integer condition!");
3012
3013 CONV(EQ);
3014 CONV(NE);
3015 CONV(GT);
3016 CONV(GE);
3017 CONV(LT);
3018 CONV(LE);
3019
3020 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
3022 }
3023#undef CONV
3024}
3025
3026// If C can be converted to a comparison against zero, adjust the operands
3027// as necessary.
3028static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3029 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3030 return;
3031
3032 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode());
3033 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
3034 return;
3035
3036 int64_t Value = ConstOp1->getSExtValue();
3037 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
3038 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
3039 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
3040 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
3041 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3042 C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
3043 }
3044}
3045
3046// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
3047// adjust the operands as necessary.
3048static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
3049 Comparison &C) {
3050 // For us to make any changes, it must a comparison between a single-use
3051 // load and a constant.
3052 if (!C.Op0.hasOneUse() ||
3053 C.Op0.getOpcode() != ISD::LOAD ||
3054 C.Op1.getOpcode() != ISD::Constant)
3055 return;
3056
3057 // We must have an 8- or 16-bit load.
3058 auto *Load = cast<LoadSDNode>(C.Op0);
3059 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
3060 if ((NumBits != 8 && NumBits != 16) ||
3061 NumBits != Load->getMemoryVT().getStoreSizeInBits())
3062 return;
3063
3064 // The load must be an extending one and the constant must be within the
3065 // range of the unextended value.
3066 auto *ConstOp1 = cast<ConstantSDNode>(C.Op1);
3067 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
3068 return;
3069 uint64_t Value = ConstOp1->getZExtValue();
3070 uint64_t Mask = (1 << NumBits) - 1;
3071 if (Load->getExtensionType() == ISD::SEXTLOAD) {
3072 // Make sure that ConstOp1 is in range of C.Op0.
3073 int64_t SignedValue = ConstOp1->getSExtValue();
3074 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
3075 return;
3076 if (C.ICmpType != SystemZICMP::SignedOnly) {
3077 // Unsigned comparison between two sign-extended values is equivalent
3078 // to unsigned comparison between two zero-extended values.
3079 Value &= Mask;
3080 } else if (NumBits == 8) {
3081 // Try to treat the comparison as unsigned, so that we can use CLI.
3082 // Adjust CCMask and Value as necessary.
3083 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
3084 // Test whether the high bit of the byte is set.
3085 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
3086 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
3087 // Test whether the high bit of the byte is clear.
3088 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
3089 else
3090 // No instruction exists for this combination.
3091 return;
3092 C.ICmpType = SystemZICMP::UnsignedOnly;
3093 }
3094 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
3095 if (Value > Mask)
3096 return;
3097 // If the constant is in range, we can use any comparison.
3098 C.ICmpType = SystemZICMP::Any;
3099 } else
3100 return;
3101
3102 // Make sure that the first operand is an i32 of the right extension type.
3103 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
3106 if (C.Op0.getValueType() != MVT::i32 ||
3107 Load->getExtensionType() != ExtType) {
3108 C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(),
3109 Load->getBasePtr(), Load->getPointerInfo(),
3110 Load->getMemoryVT(), Load->getAlign(),
3111 Load->getMemOperand()->getFlags());
3112 // Update the chain uses.
3113 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), C.Op0.getValue(1));
3114 }
3115
3116 // Make sure that the second operand is an i32 with the right value.
3117 if (C.Op1.getValueType() != MVT::i32 ||
3118 Value != ConstOp1->getZExtValue())
3119 C.Op1 = DAG.getConstant((uint32_t)Value, DL, MVT::i32);
3120}
3121
3122// Return true if Op is either an unextended load, or a load suitable
3123// for integer register-memory comparisons of type ICmpType.
3124static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
3125 auto *Load = dyn_cast<LoadSDNode>(Op.getNode());
3126 if (Load) {
3127 // There are no instructions to compare a register with a memory byte.
3128 if (Load->getMemoryVT() == MVT::i8)
3129 return false;
3130 // Otherwise decide on extension type.
3131 switch (Load->getExtensionType()) {
3132 case ISD::NON_EXTLOAD:
3133 return true;
3134 case ISD::SEXTLOAD:
3135 return ICmpType != SystemZICMP::UnsignedOnly;
3136 case ISD::ZEXTLOAD:
3137 return ICmpType != SystemZICMP::SignedOnly;
3138 default:
3139 break;
3140 }
3141 }
3142 return false;
3143}
3144
3145// Return true if it is better to swap the operands of C.
3146static bool shouldSwapCmpOperands(const Comparison &C) {
3147 // If one side of the compare is a load of the stackguard reference value,
3148 // then that load should be Op1.
3149 if (C.Op0.isMachineOpcode() &&
3150 (C.Op0.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD))
3151 return true;
3152
3153 // Leave i128 and f128 comparisons alone, since they have no memory forms.
3154 if (C.Op0.getValueType() == MVT::i128)
3155 return false;
3156 if (C.Op0.getValueType() == MVT::f128)
3157 return false;
3158
3159 // Always keep a floating-point constant second, since comparisons with
3160 // zero can use LOAD TEST and comparisons with other constants make a
3161 // natural memory operand.
3162 if (isa<ConstantFPSDNode>(C.Op1))
3163 return false;
3164
3165 // Never swap comparisons with zero since there are many ways to optimize
3166 // those later.
3167 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
3168 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
3169 return false;
3170
3171 // Also keep natural memory operands second if the loaded value is
3172 // only used here. Several comparisons have memory forms.
3173 if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse())
3174 return false;
3175
3176 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
3177 // In that case we generally prefer the memory to be second.
3178 if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) {
3179 // The only exceptions are when the second operand is a constant and
3180 // we can use things like CHHSI.
3181 if (!ConstOp1)
3182 return true;
3183 // The unsigned memory-immediate instructions can handle 16-bit
3184 // unsigned integers.
3185 if (C.ICmpType != SystemZICMP::SignedOnly &&
3186 isUInt<16>(ConstOp1->getZExtValue()))
3187 return false;
3188 // The signed memory-immediate instructions can handle 16-bit
3189 // signed integers.
3190 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
3191 isInt<16>(ConstOp1->getSExtValue()))
3192 return false;
3193 return true;
3194 }
3195
3196 // Try to promote the use of CGFR and CLGFR.
3197 unsigned Opcode0 = C.Op0.getOpcode();
3198 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
3199 return true;
3200 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
3201 return true;
3202 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
3203 C.Op0.getOperand(1).getOpcode() == ISD::Constant &&
3204 C.Op0.getConstantOperandVal(1) == 0xffffffff)
3205 return true;
3206
3207 return false;
3208}
3209
3210// Check whether C tests for equality between X and Y and whether X - Y
3211// or Y - X is also computed. In that case it's better to compare the
3212// result of the subtraction against zero.
3214 Comparison &C) {
3215 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3216 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3217 for (SDNode *N : C.Op0->users()) {
3218 if (N->getOpcode() == ISD::SUB &&
3219 ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) ||
3220 (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {
3221 // Disable the nsw and nuw flags: the backend needs to handle
3222 // overflow as well during comparison elimination.
3223 N->dropFlags(SDNodeFlags::NoWrap);
3224 C.Op0 = SDValue(N, 0);
3225 C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
3226 return;
3227 }
3228 }
3229 }
3230}
3231
3232// Check whether C compares a floating-point value with zero and if that
3233// floating-point value is also negated. In this case we can use the
3234// negation to set CC, so avoiding separate LOAD AND TEST and
3235// LOAD (NEGATIVE/COMPLEMENT) instructions.
3236static void adjustForFNeg(Comparison &C) {
3237 // This optimization is invalid for strict comparisons, since FNEG
3238 // does not raise any exceptions.
3239 if (C.Chain)
3240 return;
3241 auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1);
3242 if (C1 && C1->isZero()) {
3243 for (SDNode *N : C.Op0->users()) {
3244 if (N->getOpcode() == ISD::FNEG) {
3245 C.Op0 = SDValue(N, 0);
3246 C.CCMask = SystemZ::reverseCCMask(C.CCMask);
3247 return;
3248 }
3249 }
3250 }
3251}
3252
3253// Check whether C compares (shl X, 32) with 0 and whether X is
3254// also sign-extended. In that case it is better to test the result
3255// of the sign extension using LTGFR.
3256//
3257// This case is important because InstCombine transforms a comparison
3258// with (sext (trunc X)) into a comparison with (shl X, 32).
3259static void adjustForLTGFR(Comparison &C) {
3260 // Check for a comparison between (shl X, 32) and 0.
3261 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
3262 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
3263 auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
3264 if (C1 && C1->getZExtValue() == 32) {
3265 SDValue ShlOp0 = C.Op0.getOperand(0);
3266 // See whether X has any SIGN_EXTEND_INREG uses.
3267 for (SDNode *N : ShlOp0->users()) {
3268 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3269 cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) {
3270 C.Op0 = SDValue(N, 0);
3271 return;
3272 }
3273 }
3274 }
3275 }
3276}
3277
3278// If C compares the truncation of an extending load, try to compare
3279// the untruncated value instead. This exposes more opportunities to
3280// reuse CC.
3281static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3282 Comparison &C) {
3283 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
3284 C.Op0.getOperand(0).getOpcode() == ISD::LOAD &&
3285 C.Op1.getOpcode() == ISD::Constant &&
3286 cast<ConstantSDNode>(C.Op1)->getValueSizeInBits(0) <= 64 &&
3287 C.Op1->getAsZExtVal() == 0) {
3288 auto *L = cast<LoadSDNode>(C.Op0.getOperand(0));
3289 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3290 C.Op0.getValueSizeInBits().getFixedValue()) {
3291 unsigned Type = L->getExtensionType();
3292 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
3293 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
3294 C.Op0 = C.Op0.getOperand(0);
3295 C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
3296 }
3297 }
3298 }
3299}
3300
3301// Adjust if a given Compare is a check of the stack guard against a stack
3302// guard instance on the stack. Specifically, this checks if:
3303// - The operands are a load of the stack guard, and a load from a stack slot
3304// - The original opcode is ICMP
3305// - ICMPType is compatible with unsigned comparison.
3307 Comparison &C) {
3308
3309 // Opcode must be ICMP.
3310 if (C.Opcode != SystemZISD::ICMP)
3311 return;
3312 // ICmpType must be Unsigned or Any.
3313 if (C.ICmpType == SystemZICMP::SignedOnly)
3314 return;
3315 // Op0 must be FrameIndex Load.
3316 if (!(ISD::isNormalLoad(C.Op0.getNode()) &&
3317 dyn_cast<FrameIndexSDNode>(C.Op0.getOperand(1))))
3318 return;
3319 // Op1 must be LOAD_STACK_GUARD.
3320 if (!C.Op1.isMachineOpcode() ||
3321 C.Op1.getMachineOpcode() != SystemZ::LOAD_STACK_GUARD)
3322 return;
3323
3324 // At this point we are sure that this is a proper CMP_STACKGUARD
3325 // case, update the opcode to reflect this.
3326 C.Opcode = SystemZISD::CMP_STACKGUARD;
3327 C.Op1 = SDValue();
3328}
3329
3330// Return true if shift operation N has an in-range constant shift value.
3331// Store it in ShiftVal if so.
3332static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
3333 auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1));
3334 if (!Shift)
3335 return false;
3336
3337 uint64_t Amount = Shift->getZExtValue();
3338 if (Amount >= N.getValueSizeInBits())
3339 return false;
3340
3341 ShiftVal = Amount;
3342 return true;
3343}
3344
3345// Check whether an AND with Mask is suitable for a TEST UNDER MASK
3346// instruction and whether the CC value is descriptive enough to handle
3347// a comparison of type Opcode between the AND result and CmpVal.
3348// CCMask says which comparison result is being tested and BitSize is
3349// the number of bits in the operands. If TEST UNDER MASK can be used,
3350// return the corresponding CC mask, otherwise return 0.
3351static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
3352 uint64_t Mask, uint64_t CmpVal,
3353 unsigned ICmpType) {
3354 assert(Mask != 0 && "ANDs with zero should have been removed by now");
3355
3356 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
3357 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) &&
3358 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask))
3359 return 0;
3360
3361 // Work out the masks for the lowest and highest bits.
3363 uint64_t Low = uint64_t(1) << llvm::countr_zero(Mask);
3364
3365 // Signed ordered comparisons are effectively unsigned if the sign
3366 // bit is dropped.
3367 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
3368
3369 // Check for equality comparisons with 0, or the equivalent.
3370 if (CmpVal == 0) {
3371 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3373 if (CCMask == SystemZ::CCMASK_CMP_NE)
3375 }
3376 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
3377 if (CCMask == SystemZ::CCMASK_CMP_LT)
3379 if (CCMask == SystemZ::CCMASK_CMP_GE)
3381 }
3382 if (EffectivelyUnsigned && CmpVal < Low) {
3383 if (CCMask == SystemZ::CCMASK_CMP_LE)
3385 if (CCMask == SystemZ::CCMASK_CMP_GT)
3387 }
3388
3389 // Check for equality comparisons with the mask, or the equivalent.
3390 if (CmpVal == Mask) {
3391 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3393 if (CCMask == SystemZ::CCMASK_CMP_NE)
3395 }
3396 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
3397 if (CCMask == SystemZ::CCMASK_CMP_GT)
3399 if (CCMask == SystemZ::CCMASK_CMP_LE)
3401 }
3402 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
3403 if (CCMask == SystemZ::CCMASK_CMP_GE)
3405 if (CCMask == SystemZ::CCMASK_CMP_LT)
3407 }
3408
3409 // Check for ordered comparisons with the top bit.
3410 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
3411 if (CCMask == SystemZ::CCMASK_CMP_LE)
3413 if (CCMask == SystemZ::CCMASK_CMP_GT)
3415 }
3416 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
3417 if (CCMask == SystemZ::CCMASK_CMP_LT)
3419 if (CCMask == SystemZ::CCMASK_CMP_GE)
3421 }
3422
3423 // If there are just two bits, we can do equality checks for Low and High
3424 // as well.
3425 if (Mask == Low + High) {
3426 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
3428 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
3430 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
3432 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
3434 }
3435
3436 // Looks like we've exhausted our options.
3437 return 0;
3438}
3439
3440// See whether C can be implemented as a TEST UNDER MASK instruction.
3441// Update the arguments with the TM version if so.
3443 Comparison &C) {
3444 // Use VECTOR TEST UNDER MASK for i128 operations.
3445 if (C.Op0.getValueType() == MVT::i128) {
3446 // We can use VTM for EQ/NE comparisons of x & y against 0.
3447 if (C.Op0.getOpcode() == ISD::AND &&
3448 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3449 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
3450 auto *Mask = dyn_cast<ConstantSDNode>(C.Op1);
3451 if (Mask && Mask->getAPIntValue() == 0) {
3452 C.Opcode = SystemZISD::VTM;
3453 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(1));
3454 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(0));
3455 C.CCValid = SystemZ::CCMASK_VCMP;
3456 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3457 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3458 else
3459 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3460 }
3461 }
3462 return;
3463 }
3464
3465 // Check that we have a comparison with a constant.
3466 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
3467 if (!ConstOp1)
3468 return;
3469 uint64_t CmpVal = ConstOp1->getZExtValue();
3470
3471 // Check whether the nonconstant input is an AND with a constant mask.
3472 Comparison NewC(C);
3473 uint64_t MaskVal;
3474 ConstantSDNode *Mask = nullptr;
3475 if (C.Op0.getOpcode() == ISD::AND) {
3476 NewC.Op0 = C.Op0.getOperand(0);
3477 NewC.Op1 = C.Op0.getOperand(1);
3478 Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
3479 if (!Mask)
3480 return;
3481 MaskVal = Mask->getZExtValue();
3482 } else {
3483 // There is no instruction to compare with a 64-bit immediate
3484 // so use TMHH instead if possible. We need an unsigned ordered
3485 // comparison with an i64 immediate.
3486 if (NewC.Op0.getValueType() != MVT::i64 ||
3487 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
3488 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
3489 NewC.ICmpType == SystemZICMP::SignedOnly)
3490 return;
3491 // Convert LE and GT comparisons into LT and GE.
3492 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
3493 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
3494 if (CmpVal == uint64_t(-1))
3495 return;
3496 CmpVal += 1;
3497 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3498 }
3499 // If the low N bits of Op1 are zero than the low N bits of Op0 can
3500 // be masked off without changing the result.
3501 MaskVal = -(CmpVal & -CmpVal);
3502 NewC.ICmpType = SystemZICMP::UnsignedOnly;
3503 }
3504 if (!MaskVal)
3505 return;
3506
3507 // Check whether the combination of mask, comparison value and comparison
3508 // type are suitable.
3509 unsigned BitSize = NewC.Op0.getValueSizeInBits();
3510 unsigned NewCCMask, ShiftVal;
3511 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3512 NewC.Op0.getOpcode() == ISD::SHL &&
3513 isSimpleShift(NewC.Op0, ShiftVal) &&
3514 (MaskVal >> ShiftVal != 0) &&
3515 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3516 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
3517 MaskVal >> ShiftVal,
3518 CmpVal >> ShiftVal,
3519 SystemZICMP::Any))) {
3520 NewC.Op0 = NewC.Op0.getOperand(0);
3521 MaskVal >>= ShiftVal;
3522 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3523 NewC.Op0.getOpcode() == ISD::SRL &&
3524 isSimpleShift(NewC.Op0, ShiftVal) &&
3525 (MaskVal << ShiftVal != 0) &&
3526 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3527 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
3528 MaskVal << ShiftVal,
3529 CmpVal << ShiftVal,
3531 NewC.Op0 = NewC.Op0.getOperand(0);
3532 MaskVal <<= ShiftVal;
3533 } else {
3534 NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal,
3535 NewC.ICmpType);
3536 if (!NewCCMask)
3537 return;
3538 }
3539
3540 // Go ahead and make the change.
3541 C.Opcode = SystemZISD::TM;
3542 C.Op0 = NewC.Op0;
3543 if (Mask && Mask->getZExtValue() == MaskVal)
3544 C.Op1 = SDValue(Mask, 0);
3545 else
3546 C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
3547 C.CCValid = SystemZ::CCMASK_TM;
3548 C.CCMask = NewCCMask;
3549}
3550
3551// Implement i128 comparison in vector registers.
3552static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3553 Comparison &C) {
3554 if (C.Opcode != SystemZISD::ICMP)
3555 return;
3556 if (C.Op0.getValueType() != MVT::i128)
3557 return;
3558
3559 // Recognize vector comparison reductions.
3560 if ((C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3561 C.CCMask == SystemZ::CCMASK_CMP_NE) &&
3562 (isNullConstant(C.Op1) || isAllOnesConstant(C.Op1))) {
3563 bool CmpEq = C.CCMask == SystemZ::CCMASK_CMP_EQ;
3564 bool CmpNull = isNullConstant(C.Op1);
3565 SDValue Src = peekThroughBitcasts(C.Op0);
3566 if (Src.hasOneUse() && isBitwiseNot(Src)) {
3567 Src = Src.getOperand(0);
3568 CmpNull = !CmpNull;
3569 }
3570 unsigned Opcode = 0;
3571 if (Src.hasOneUse()) {
3572 switch (Src.getOpcode()) {
3573 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES; break;
3574 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS; break;
3575 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS; break;
3576 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES; break;
3577 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS; break;
3578 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES; break;
3579 default: break;
3580 }
3581 }
3582 if (Opcode) {
3583 C.Opcode = Opcode;
3584 C.Op0 = Src->getOperand(0);
3585 C.Op1 = Src->getOperand(1);
3586 C.CCValid = SystemZ::CCMASK_VCMP;
3588 if (!CmpEq)
3589 C.CCMask ^= C.CCValid;
3590 return;
3591 }
3592 }
3593
3594 // Everything below here is not useful if we have native i128 compares.
3595 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3596 return;
3597
3598 // (In-)Equality comparisons can be implemented via VCEQGS.
3599 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3600 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3601 C.Opcode = SystemZISD::VICMPES;
3602 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op0);
3603 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op1);
3604 C.CCValid = SystemZ::CCMASK_VCMP;
3605 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3606 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3607 else
3608 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3609 return;
3610 }
3611
3612 // Normalize other comparisons to GT.
3613 bool Swap = false, Invert = false;
3614 switch (C.CCMask) {
3615 case SystemZ::CCMASK_CMP_GT: break;
3616 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
3617 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
3618 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
3619 default: llvm_unreachable("Invalid integer condition!");
3620 }
3621 if (Swap)
3622 std::swap(C.Op0, C.Op1);
3623
3624 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3625 C.Opcode = SystemZISD::UCMP128HI;
3626 else
3627 C.Opcode = SystemZISD::SCMP128HI;
3628 C.CCValid = SystemZ::CCMASK_ANY;
3629 C.CCMask = SystemZ::CCMASK_1;
3630
3631 if (Invert)
3632 C.CCMask ^= C.CCValid;
3633}
3634
3635// See whether the comparison argument contains a redundant AND
3636// and remove it if so. This sometimes happens due to the generic
3637// BRCOND expansion.
3639 Comparison &C) {
3640 if (C.Op0.getOpcode() != ISD::AND)
3641 return;
3642 auto *Mask = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
3643 if (!Mask || Mask->getValueSizeInBits(0) > 64)
3644 return;
3645 KnownBits Known = DAG.computeKnownBits(C.Op0.getOperand(0));
3646 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3647 return;
3648
3649 C.Op0 = C.Op0.getOperand(0);
3650}
3651
3652// Return a Comparison that tests the condition-code result of intrinsic
3653// node Call against constant integer CC using comparison code Cond.
3654// Opcode is the opcode of the SystemZISD operation for the intrinsic
3655// and CCValid is the set of possible condition-code results.
3656static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3657 SDValue Call, unsigned CCValid, uint64_t CC,
3659 Comparison C(Call, SDValue(), SDValue());
3660 C.Opcode = Opcode;
3661 C.CCValid = CCValid;
3662 if (Cond == ISD::SETEQ)
3663 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3664 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3665 else if (Cond == ISD::SETNE)
3666 // ...and the inverse of that.
3667 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3668 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3669 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3670 // always true for CC>3.
3671 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3672 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3673 // ...and the inverse of that.
3674 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3675 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3676 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3677 // always true for CC>3.
3678 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3679 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3680 // ...and the inverse of that.
3681 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3682 else
3683 llvm_unreachable("Unexpected integer comparison type");
3684 C.CCMask &= CCValid;
3685 return C;
3686}
3687
3688// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
3689static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3690 ISD::CondCode Cond, const SDLoc &DL,
3691 SDValue Chain = SDValue(),
3692 bool IsSignaling = false) {
3693 if (CmpOp1.getOpcode() == ISD::Constant) {
3694 assert(!Chain);
3695 unsigned Opcode, CCValid;
3696 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3697 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) &&
3698 isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid))
3699 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid,
3700 CmpOp1->getAsZExtVal(), Cond);
3701 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3702 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3703 isIntrinsicWithCC(CmpOp0, Opcode, CCValid))
3704 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid,
3705 CmpOp1->getAsZExtVal(), Cond);
3706 }
3707 Comparison C(CmpOp0, CmpOp1, Chain);
3708 C.CCMask = CCMaskForCondCode(Cond);
3709 if (C.Op0.getValueType().isFloatingPoint()) {
3710 C.CCValid = SystemZ::CCMASK_FCMP;
3711 if (!C.Chain)
3712 C.Opcode = SystemZISD::FCMP;
3713 else if (!IsSignaling)
3714 C.Opcode = SystemZISD::STRICT_FCMP;
3715 else
3716 C.Opcode = SystemZISD::STRICT_FCMPS;
3718 } else {
3719 assert(!C.Chain);
3720 C.CCValid = SystemZ::CCMASK_ICMP;
3721 C.Opcode = SystemZISD::ICMP;
3722 // Choose the type of comparison. Equality and inequality tests can
3723 // use either signed or unsigned comparisons. The choice also doesn't
3724 // matter if both sign bits are known to be clear. In those cases we
3725 // want to give the main isel code the freedom to choose whichever
3726 // form fits best.
3727 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3728 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3729 (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1)))
3730 C.ICmpType = SystemZICMP::Any;
3731 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3732 C.ICmpType = SystemZICMP::UnsignedOnly;
3733 else
3734 C.ICmpType = SystemZICMP::SignedOnly;
3735 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3736 adjustForRedundantAnd(DAG, DL, C);
3737 adjustZeroCmp(DAG, DL, C);
3738 adjustSubwordCmp(DAG, DL, C);
3739 adjustForSubtraction(DAG, DL, C);
3741 adjustICmpTruncate(DAG, DL, C);
3742 }
3743
3744 if (shouldSwapCmpOperands(C)) {
3745 std::swap(C.Op0, C.Op1);
3746 C.CCMask = SystemZ::reverseCCMask(C.CCMask);
3747 }
3748
3750 adjustICmp128(DAG, DL, C);
3752 return C;
3753}
3754
3755// Emit the comparison instruction described by C.
3756static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3757 if (!C.Op1.getNode()) {
3758 if (C.Opcode == SystemZISD::CMP_STACKGUARD)
3759 return DAG.getNode(SystemZISD::CMP_STACKGUARD, DL, MVT::i32, C.Op0);
3760 SDNode *Node;
3761 switch (C.Op0.getOpcode()) {
3763 Node = emitIntrinsicWithCCAndChain(DAG, C.Op0, C.Opcode);
3764 return SDValue(Node, 0);
3766 Node = emitIntrinsicWithCC(DAG, C.Op0, C.Opcode);
3767 return SDValue(Node, Node->getNumValues() - 1);
3768 default:
3769 llvm_unreachable("Invalid comparison operands");
3770 }
3771 }
3772 if (C.Opcode == SystemZISD::ICMP)
3773 return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1,
3774 DAG.getTargetConstant(C.ICmpType, DL, MVT::i32));
3775 if (C.Opcode == SystemZISD::TM) {
3776 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3778 return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
3779 DAG.getTargetConstant(RegisterOnly, DL, MVT::i32));
3780 }
3781 if (C.Opcode == SystemZISD::VICMPES ||
3782 C.Opcode == SystemZISD::VICMPHS ||
3783 C.Opcode == SystemZISD::VICMPHLS ||
3784 C.Opcode == SystemZISD::VFCMPES ||
3785 C.Opcode == SystemZISD::VFCMPHS ||
3786 C.Opcode == SystemZISD::VFCMPHES) {
3787 EVT IntVT = C.Op0.getValueType().changeVectorElementTypeToInteger();
3788 SDVTList VTs = DAG.getVTList(IntVT, MVT::i32);
3789 SDValue Val = DAG.getNode(C.Opcode, DL, VTs, C.Op0, C.Op1);
3790 return SDValue(Val.getNode(), 1);
3791 }
3792 if (C.Chain) {
3793 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
3794 return DAG.getNode(C.Opcode, DL, VTs, C.Chain, C.Op0, C.Op1);
3795 }
3796 return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
3797}
3798
3799// Implement a 32-bit *MUL_LOHI operation by extending both operands to
3800// 64 bits. Extend is the extension type to use. Store the high part
3801// in Hi and the low part in Lo.
3802static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3803 SDValue Op0, SDValue Op1, SDValue &Hi,
3804 SDValue &Lo) {
3805 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
3806 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
3807 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
3808 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
3809 DAG.getConstant(32, DL, MVT::i64));
3810 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
3811 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
3812}
3813
3814// Lower a binary operation that produces two VT results, one in each
3815// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3816// and Opcode performs the GR128 operation. Store the even register result
3817// in Even and the odd register result in Odd.
3818static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3819 unsigned Opcode, SDValue Op0, SDValue Op1,
3820 SDValue &Even, SDValue &Odd) {
3821 SDValue Result = DAG.getNode(Opcode, DL, MVT::v2i64, Op0, Op1);
3822 bool Is32Bit = is32Bit(VT);
3823 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
3824 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
3825}
3826
3827// Return an i32 value that is 1 if the CC value produced by CCReg is
3828// in the mask CCMask and 0 otherwise. CC is known to have a value
3829// in CCValid, so other values can be ignored.
3830static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3831 unsigned CCValid, unsigned CCMask) {
3832 SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32),
3833 DAG.getConstant(0, DL, MVT::i32),
3834 DAG.getTargetConstant(CCValid, DL, MVT::i32),
3835 DAG.getTargetConstant(CCMask, DL, MVT::i32), CCReg};
3836 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
3837}
3838
3839// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3840// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3841// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3842// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3843// floating-point comparisons.
3846 switch (CC) {
3847 case ISD::SETOEQ:
3848 case ISD::SETEQ:
3849 switch (Mode) {
3850 case CmpMode::Int: return SystemZISD::VICMPE;
3851 case CmpMode::FP: return SystemZISD::VFCMPE;
3852 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3853 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3854 }
3855 llvm_unreachable("Bad mode");
3856
3857 case ISD::SETOGE:
3858 case ISD::SETGE:
3859 switch (Mode) {
3860 case CmpMode::Int: return 0;
3861 case CmpMode::FP: return SystemZISD::VFCMPHE;
3862 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3863 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3864 }
3865 llvm_unreachable("Bad mode");
3866
3867 case ISD::SETOGT:
3868 case ISD::SETGT:
3869 switch (Mode) {
3870 case CmpMode::Int: return SystemZISD::VICMPH;
3871 case CmpMode::FP: return SystemZISD::VFCMPH;
3872 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3873 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3874 }
3875 llvm_unreachable("Bad mode");
3876
3877 case ISD::SETUGT:
3878 switch (Mode) {
3879 case CmpMode::Int: return SystemZISD::VICMPHL;
3880 case CmpMode::FP: return 0;
3881 case CmpMode::StrictFP: return 0;
3882 case CmpMode::SignalingFP: return 0;
3883 }
3884 llvm_unreachable("Bad mode");
3885
3886 default:
3887 return 0;
3888 }
3889}
3890
3891// Return the SystemZISD vector comparison operation for CC or its inverse,
3892// or 0 if neither can be done directly. Indicate in Invert whether the
3893// result is for the inverse of CC. Mode is as above.
3895 bool &Invert) {
3896 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3897 Invert = false;
3898 return Opcode;
3899 }
3900
3901 CC = ISD::getSetCCInverse(CC, Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3902 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3903 Invert = true;
3904 return Opcode;
3905 }
3906
3907 return 0;
3908}
3909
3910// Return a v2f64 that contains the extended form of elements Start and Start+1
3911// of v4f32 value Op. If Chain is nonnull, return the strict form.
3912static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3913 SDValue Op, SDValue Chain) {
3914 int Mask[] = { Start, -1, Start + 1, -1 };
3915 Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
3916 if (Chain) {
3917 SDVTList VTs = DAG.getVTList(MVT::v2f64, MVT::Other);
3918 return DAG.getNode(SystemZISD::STRICT_VEXTEND, DL, VTs, Chain, Op);
3919 }
3920 return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
3921}
3922
3923// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3924// producing a result of type VT. If Chain is nonnull, return the strict form.
3925SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3926 const SDLoc &DL, EVT VT,
3927 SDValue CmpOp0,
3928 SDValue CmpOp1,
3929 SDValue Chain) const {
3930 // There is no hardware support for v4f32 (unless we have the vector
3931 // enhancements facility 1), so extend the vector into two v2f64s
3932 // and compare those.
3933 if (CmpOp0.getValueType() == MVT::v4f32 &&
3934 !Subtarget.hasVectorEnhancements1()) {
3935 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0, Chain);
3936 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0, Chain);
3937 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1, Chain);
3938 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1, Chain);
3939 if (Chain) {
3940 SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::Other);
3941 SDValue HRes = DAG.getNode(Opcode, DL, VTs, Chain, H0, H1);
3942 SDValue LRes = DAG.getNode(Opcode, DL, VTs, Chain, L0, L1);
3943 SDValue Res = DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
3944 SDValue Chains[6] = { H0.getValue(1), L0.getValue(1),
3945 H1.getValue(1), L1.getValue(1),
3946 HRes.getValue(1), LRes.getValue(1) };
3947 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
3948 SDValue Ops[2] = { Res, NewChain };
3949 return DAG.getMergeValues(Ops, DL);
3950 }
3951 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
3952 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
3953 return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
3954 }
3955 if (Chain) {
3956 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
3957 return DAG.getNode(Opcode, DL, VTs, Chain, CmpOp0, CmpOp1);
3958 }
3959 return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
3960}
3961
3962// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3963// an integer mask of type VT. If Chain is nonnull, we have a strict
3964// floating-point comparison. If in addition IsSignaling is true, we have
3965// a strict signaling floating-point comparison.
3966SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3967 const SDLoc &DL, EVT VT,
3969 SDValue CmpOp0,
3970 SDValue CmpOp1,
3971 SDValue Chain,
3972 bool IsSignaling) const {
3973 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3974 assert (!Chain || IsFP);
3975 assert (!IsSignaling || Chain);
3976 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3977 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3978 bool Invert = false;
3979 SDValue Cmp;
3980 switch (CC) {
3981 // Handle tests for order using (or (ogt y x) (oge x y)).
3982 case ISD::SETUO:
3983 Invert = true;
3984 [[fallthrough]];
3985 case ISD::SETO: {
3986 assert(IsFP && "Unexpected integer comparison");
3987 SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
3988 DL, VT, CmpOp1, CmpOp0, Chain);
3989 SDValue GE = getVectorCmp(DAG, getVectorComparison(ISD::SETOGE, Mode),
3990 DL, VT, CmpOp0, CmpOp1, Chain);
3991 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
3992 if (Chain)
3993 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3994 LT.getValue(1), GE.getValue(1));
3995 break;
3996 }
3997
3998 // Handle <> tests using (or (ogt y x) (ogt x y)).
3999 case ISD::SETUEQ:
4000 Invert = true;
4001 [[fallthrough]];
4002 case ISD::SETONE: {
4003 assert(IsFP && "Unexpected integer comparison");
4004 SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
4005 DL, VT, CmpOp1, CmpOp0, Chain);
4006 SDValue GT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
4007 DL, VT, CmpOp0, CmpOp1, Chain);
4008 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
4009 if (Chain)
4010 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
4011 LT.getValue(1), GT.getValue(1));
4012 break;
4013 }
4014
4015 // Otherwise a single comparison is enough. It doesn't really
4016 // matter whether we try the inversion or the swap first, since
4017 // there are no cases where both work.
4018 default:
4019 // Optimize sign-bit comparisons to signed compares.
4020 if (Mode == CmpMode::Int && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
4022 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4023 APInt Mask;
4024 if (CmpOp0.getOpcode() == ISD::AND &&
4025 ISD::isConstantSplatVector(CmpOp0.getOperand(1), Mask) &&
4026 Mask == APInt::getSignMask(EltSize)) {
4028 CmpOp0 = CmpOp0.getOperand(0);
4029 }
4030 }
4031 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
4032 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
4033 else {
4035 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
4036 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0, Chain);
4037 else
4038 llvm_unreachable("Unhandled comparison");
4039 }
4040 if (Chain)
4041 Chain = Cmp.getValue(1);
4042 break;
4043 }
4044 if (Invert) {
4045 SDValue Mask =
4046 DAG.getSplatBuildVector(VT, DL, DAG.getAllOnesConstant(DL, MVT::i64));
4047 Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
4048 }
4049 if (Chain && Chain.getNode() != Cmp.getNode()) {
4050 SDValue Ops[2] = { Cmp, Chain };
4051 Cmp = DAG.getMergeValues(Ops, DL);
4052 }
4053 return Cmp;
4054}
4055
4056SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
4057 SelectionDAG &DAG) const {
4058 SDValue CmpOp0 = Op.getOperand(0);
4059 SDValue CmpOp1 = Op.getOperand(1);
4060 ISD::CondCode CC = Op.getOperandCondCode(2);
4061 SDLoc DL(Op);
4062 EVT VT = Op.getValueType();
4063 if (VT.isVector())
4064 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
4065
4066 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
4067 SDValue CCReg = emitCmp(DAG, DL, C);
4068 return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
4069}
4070
4071SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
4072 SelectionDAG &DAG,
4073 bool IsSignaling) const {
4074 SDValue Chain = Op.getOperand(0);
4075 SDValue CmpOp0 = Op.getOperand(1);
4076 SDValue CmpOp1 = Op.getOperand(2);
4077 ISD::CondCode CC = Op.getOperandCondCode(3);
4078 SDLoc DL(Op);
4079 EVT VT = Op.getNode()->getValueType(0);
4080 if (VT.isVector()) {
4081 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
4082 Chain, IsSignaling);
4083 return Res.getValue(Op.getResNo());
4084 }
4085
4086 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL, Chain, IsSignaling));
4087 SDValue CCReg = emitCmp(DAG, DL, C);
4088 CCReg->setFlags(Op->getFlags());
4089 SDValue Result = emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
4090 SDValue Ops[2] = { Result, CCReg.getValue(1) };
4091 return DAG.getMergeValues(Ops, DL);
4092}
4093
4094SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4095 ISD::CondCode CC = Op.getOperandCondCode(1);
4096 SDValue CmpOp0 = Op.getOperand(2);
4097 SDValue CmpOp1 = Op.getOperand(3);
4098 SDValue Dest = Op.getOperand(4);
4099 SDLoc DL(Op);
4100
4101 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
4102 SDValue CCReg = emitCmp(DAG, DL, C);
4103 return DAG.getNode(
4104 SystemZISD::BR_CCMASK, DL, Op.getValueType(), Op.getOperand(0),
4105 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
4106 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
4107}
4108
4109// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
4110// allowing Pos and Neg to be wider than CmpOp.
4111static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
4112 return (Neg.getOpcode() == ISD::SUB &&
4113 Neg.getOperand(0).getOpcode() == ISD::Constant &&
4114 Neg.getConstantOperandVal(0) == 0 && Neg.getOperand(1) == Pos &&
4115 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
4116 Pos.getOperand(0) == CmpOp)));
4117}
4118
4119// Return the absolute or negative absolute of Op; IsNegative decides which.
4121 bool IsNegative) {
4122 Op = DAG.getNode(ISD::ABS, DL, Op.getValueType(), Op);
4123 if (IsNegative)
4124 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
4125 DAG.getConstant(0, DL, Op.getValueType()), Op);
4126 return Op;
4127}
4128
4130 Comparison C, SDValue TrueOp, SDValue FalseOp) {
4131 EVT VT = MVT::i128;
4132 unsigned Op;
4133
4134 if (C.CCMask == SystemZ::CCMASK_CMP_NE ||
4135 C.CCMask == SystemZ::CCMASK_CMP_GE ||
4136 C.CCMask == SystemZ::CCMASK_CMP_LE) {
4137 std::swap(TrueOp, FalseOp);
4138 C.CCMask ^= C.CCValid;
4139 }
4140 if (C.CCMask == SystemZ::CCMASK_CMP_LT) {
4141 std::swap(C.Op0, C.Op1);
4142 C.CCMask = SystemZ::CCMASK_CMP_GT;
4143 }
4144 switch (C.CCMask) {
4146 Op = SystemZISD::VICMPE;
4147 break;
4149 if (C.ICmpType == SystemZICMP::UnsignedOnly)
4150 Op = SystemZISD::VICMPHL;
4151 else
4152 Op = SystemZISD::VICMPH;
4153 break;
4154 default:
4155 llvm_unreachable("Unhandled comparison");
4156 break;
4157 }
4158
4159 SDValue Mask = DAG.getNode(Op, DL, VT, C.Op0, C.Op1);
4160 TrueOp = DAG.getNode(ISD::AND, DL, VT, TrueOp, Mask);
4161 FalseOp = DAG.getNode(ISD::AND, DL, VT, FalseOp, DAG.getNOT(DL, Mask, VT));
4162 return DAG.getNode(ISD::OR, DL, VT, TrueOp, FalseOp);
4163}
4164
4165SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
4166 SelectionDAG &DAG) const {
4167 SDValue CmpOp0 = Op.getOperand(0);
4168 SDValue CmpOp1 = Op.getOperand(1);
4169 SDValue TrueOp = Op.getOperand(2);
4170 SDValue FalseOp = Op.getOperand(3);
4171 ISD::CondCode CC = Op.getOperandCondCode(4);
4172 SDLoc DL(Op);
4173
4174 // SELECT_CC involving f16 will not have the cmp-ops promoted by the
4175 // legalizer, as it will be handled according to the type of the resulting
4176 // value. Extend them here if needed.
4177 if (CmpOp0.getSimpleValueType() == MVT::f16) {
4178 CmpOp0 = DAG.getFPExtendOrRound(CmpOp0, SDLoc(CmpOp0), MVT::f32);
4179 CmpOp1 = DAG.getFPExtendOrRound(CmpOp1, SDLoc(CmpOp1), MVT::f32);
4180 }
4181
4182 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
4183
4184 // Check for absolute and negative-absolute selections, including those
4185 // where the comparison value is sign-extended (for LPGFR and LNGFR).
4186 // This check supplements the one in DAGCombiner.
4187 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
4188 C.CCMask != SystemZ::CCMASK_CMP_NE &&
4189 C.Op1.getOpcode() == ISD::Constant &&
4190 cast<ConstantSDNode>(C.Op1)->getValueSizeInBits(0) <= 64 &&
4191 C.Op1->getAsZExtVal() == 0) {
4192 if (isAbsolute(C.Op0, TrueOp, FalseOp))
4193 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
4194 if (isAbsolute(C.Op0, FalseOp, TrueOp))
4195 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
4196 }
4197
4198 if (Subtarget.hasVectorEnhancements3() &&
4199 C.Opcode == SystemZISD::ICMP &&
4200 C.Op0.getValueType() == MVT::i128 &&
4201 TrueOp.getValueType() == MVT::i128) {
4202 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
4203 }
4204
4205 SDValue CCReg = emitCmp(DAG, DL, C);
4206 SDValue Ops[] = {TrueOp, FalseOp,
4207 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
4208 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), CCReg};
4209
4210 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
4211}
4212
4213SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
4214 SelectionDAG &DAG) const {
4215 SDLoc DL(Node);
4216 const GlobalValue *GV = Node->getGlobal();
4217 int64_t Offset = Node->getOffset();
4218 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4220
4222 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
4223 if (isInt<32>(Offset)) {
4224 // Assign anchors at 1<<12 byte boundaries.
4225 uint64_t Anchor = Offset & ~uint64_t(0xfff);
4226 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor);
4227 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4228
4229 // The offset can be folded into the address if it is aligned to a
4230 // halfword.
4231 Offset -= Anchor;
4232 if (Offset != 0 && (Offset & 1) == 0) {
4233 SDValue Full =
4234 DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
4235 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result);
4236 Offset = 0;
4237 }
4238 } else {
4239 // Conservatively load a constant offset greater than 32 bits into a
4240 // register below.
4241 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT);
4242 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4243 }
4244 } else if (Subtarget.isTargetELF()) {
4245 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT);
4246 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4247 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
4249 } else if (Subtarget.isTargetzOS()) {
4250 Result = getADAEntry(DAG, GV, DL, PtrVT);
4251 } else
4252 llvm_unreachable("Unexpected Subtarget");
4253
4254 // If there was a non-zero offset that we didn't fold, create an explicit
4255 // addition for it.
4256 if (Offset != 0)
4257 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
4258 DAG.getSignedConstant(Offset, DL, PtrVT));
4259
4260 return Result;
4261}
4262
4263SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
4264 SelectionDAG &DAG,
4265 unsigned Opcode,
4266 SDValue GOTOffset) const {
4267 SDLoc DL(Node);
4268 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4269 SDValue Chain = DAG.getEntryNode();
4270 SDValue Glue;
4271
4274 report_fatal_error("In GHC calling convention TLS is not supported");
4275
4276 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
4277 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
4278 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
4279 Glue = Chain.getValue(1);
4280 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
4281 Glue = Chain.getValue(1);
4282
4283 // The first call operand is the chain and the second is the TLS symbol.
4285 Ops.push_back(Chain);
4286 Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,
4287 Node->getValueType(0),
4288 0, 0));
4289
4290 // Add argument registers to the end of the list so that they are
4291 // known live into the call.
4292 Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT));
4293 Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT));
4294
4295 // Add a register mask operand representing the call-preserved registers.
4296 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4297 const uint32_t *Mask =
4298 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
4299 assert(Mask && "Missing call preserved mask for calling convention");
4300 Ops.push_back(DAG.getRegisterMask(Mask));
4301
4302 // Glue the call to the argument copies.
4303 Ops.push_back(Glue);
4304
4305 // Emit the call.
4306 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
4307 Chain = DAG.getNode(Opcode, DL, NodeTys, Ops);
4308 Glue = Chain.getValue(1);
4309
4310 // Copy the return value from %r2.
4311 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
4312}
4313
4314SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
4315 SelectionDAG &DAG) const {
4316 SDValue Chain = DAG.getEntryNode();
4317 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4318
4319 // The high part of the thread pointer is in access register 0.
4320 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
4321 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
4322
4323 // The low part of the thread pointer is in access register 1.
4324 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
4325 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
4326
4327 // Merge them into a single 64-bit address.
4328 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
4329 DAG.getConstant(32, DL, PtrVT));
4330 return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
4331}
4332
4333SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
4334 SelectionDAG &DAG) const {
4335 if (DAG.getTarget().useEmulatedTLS())
4336 return LowerToTLSEmulatedModel(Node, DAG);
4337 SDLoc DL(Node);
4338 const GlobalValue *GV = Node->getGlobal();
4339 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4340 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
4341
4344 report_fatal_error("In GHC calling convention TLS is not supported");
4345
4346 SDValue TP = lowerThreadPointer(DL, DAG);
4347
4348 // Get the offset of GA from the thread pointer, based on the TLS model.
4350 switch (model) {
4352 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
4353 SystemZConstantPoolValue *CPV =
4355
4356 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4357 Offset = DAG.getLoad(
4358 PtrVT, DL, DAG.getEntryNode(), Offset,
4360
4361 // Call __tls_get_offset to retrieve the offset.
4362 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset);
4363 break;
4364 }
4365
4367 // Load the GOT offset of the module ID.
4368 SystemZConstantPoolValue *CPV =
4370
4371 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4372 Offset = DAG.getLoad(
4373 PtrVT, DL, DAG.getEntryNode(), Offset,
4375
4376 // Call __tls_get_offset to retrieve the module base offset.
4377 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset);
4378
4379 // Note: The SystemZLDCleanupPass will remove redundant computations
4380 // of the module base offset. Count total number of local-dynamic
4381 // accesses to trigger execution of that pass.
4382 SystemZMachineFunctionInfo* MFI =
4383 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4385
4386 // Add the per-symbol offset.
4388
4389 SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4390 DTPOffset = DAG.getLoad(
4391 PtrVT, DL, DAG.getEntryNode(), DTPOffset,
4393
4394 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset);
4395 break;
4396 }
4397
4398 case TLSModel::InitialExec: {
4399 // Load the offset from the GOT.
4400 Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
4402 Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset);
4403 Offset =
4404 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset,
4406 break;
4407 }
4408
4409 case TLSModel::LocalExec: {
4410 // Force the offset into the constant pool and load it from there.
4411 SystemZConstantPoolValue *CPV =
4413
4414 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
4415 Offset = DAG.getLoad(
4416 PtrVT, DL, DAG.getEntryNode(), Offset,
4418 break;
4419 }
4420 }
4421
4422 // Add the base and offset together.
4423 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset);
4424}
4425
4426SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
4427 SelectionDAG &DAG) const {
4428 SDLoc DL(Node);
4429 const BlockAddress *BA = Node->getBlockAddress();
4430 int64_t Offset = Node->getOffset();
4431 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4432
4433 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
4434 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4435 return Result;
4436}
4437
4438SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
4439 SelectionDAG &DAG) const {
4440 SDLoc DL(JT);
4441 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4442 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
4443
4444 // Use LARL to load the address of the table.
4445 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4446}
4447
4448SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
4449 SelectionDAG &DAG) const {
4450 SDLoc DL(CP);
4451 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4452
4455 Result =
4456 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CP->getAlign());
4457 else
4458 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlign(),
4459 CP->getOffset());
4460
4461 // Use LARL to load the address of the constant pool entry.
4462 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
4463}
4464
4465SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
4466 SelectionDAG &DAG) const {
4467 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4469 MachineFrameInfo &MFI = MF.getFrameInfo();
4470 MFI.setFrameAddressIsTaken(true);
4471
4472 SDLoc DL(Op);
4473 unsigned Depth = Op.getConstantOperandVal(0);
4474 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4475
4476 // By definition, the frame address is the address of the back chain. (In
4477 // the case of packed stack without backchain, return the address where the
4478 // backchain would have been stored. This will either be an unused space or
4479 // contain a saved register).
4480 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4481 SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
4482
4483 if (Depth > 0) {
4484 // FIXME The frontend should detect this case.
4485 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4486 report_fatal_error("Unsupported stack frame traversal count");
4487
4488 SDValue Offset = DAG.getConstant(TFL->getBackchainOffset(MF), DL, PtrVT);
4489 while (Depth--) {
4490 BackChain = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), BackChain,
4491 MachinePointerInfo());
4492 BackChain = DAG.getNode(ISD::ADD, DL, PtrVT, BackChain, Offset);
4493 }
4494 }
4495
4496 return BackChain;
4497}
4498
4499SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
4500 SelectionDAG &DAG) const {
4502 MachineFrameInfo &MFI = MF.getFrameInfo();
4503 MFI.setReturnAddressIsTaken(true);
4504
4505 SDLoc DL(Op);
4506 unsigned Depth = Op.getConstantOperandVal(0);
4507 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4508
4509 if (Depth > 0) {
4510 // FIXME The frontend should detect this case.
4511 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4512 report_fatal_error("Unsupported stack frame traversal count");
4513
4514 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4515 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4516 int Offset = TFL->getReturnAddressOffset(MF);
4517 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, FrameAddr,
4518 DAG.getSignedConstant(Offset, DL, PtrVT));
4519 return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr,
4520 MachinePointerInfo());
4521 }
4522
4523 // Return R14D (Elf) / R7D (XPLINK), which has the return address. Mark it an
4524 // implicit live-in.
4525 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4527 &SystemZ::GR64BitRegClass);
4528 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT);
4529}
4530
4531SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
4532 SelectionDAG &DAG) const {
4533 SDLoc DL(Op);
4534 SDValue In = Op.getOperand(0);
4535 EVT InVT = In.getValueType();
4536 EVT ResVT = Op.getValueType();
4537
4538 // Convert loads directly. This is normally done by DAGCombiner,
4539 // but we need this case for bitcasts that are created during lowering
4540 // and which are then lowered themselves.
4541 if (auto *LoadN = dyn_cast<LoadSDNode>(In))
4542 if (ISD::isNormalLoad(LoadN)) {
4543 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
4544 LoadN->getBasePtr(), LoadN->getMemOperand());
4545 // Update the chain uses.
4546 DAG.ReplaceAllUsesOfValueWith(SDValue(LoadN, 1), NewLoad.getValue(1));
4547 return NewLoad;
4548 }
4549
4550 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4551 SDValue In64;
4552 if (Subtarget.hasHighWord()) {
4553 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL,
4554 MVT::i64);
4555 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
4556 MVT::i64, SDValue(U64, 0), In);
4557 } else {
4558 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
4559 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
4560 DAG.getConstant(32, DL, MVT::i64));
4561 }
4562 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
4563 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32,
4564 DL, MVT::f32, Out64);
4565 }
4566 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4567 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
4568 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
4569 MVT::f64, SDValue(U64, 0), In);
4570 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
4571 if (Subtarget.hasHighWord())
4572 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
4573 MVT::i32, Out64);
4574 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
4575 DAG.getConstant(32, DL, MVT::i64));
4576 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
4577 }
4578 llvm_unreachable("Unexpected bitcast combination");
4579}
4580
4581SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
4582 SelectionDAG &DAG) const {
4583
4584 if (Subtarget.isTargetXPLINK64())
4585 return lowerVASTART_XPLINK(Op, DAG);
4586 else
4587 return lowerVASTART_ELF(Op, DAG);
4588}
4589
4590SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
4591 SelectionDAG &DAG) const {
4593 SystemZMachineFunctionInfo *FuncInfo =
4594 MF.getInfo<SystemZMachineFunctionInfo>();
4595
4596 SDLoc DL(Op);
4597
4598 // vastart just stores the address of the VarArgsFrameIndex slot into the
4599 // memory location argument.
4600 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4601 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4602 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4603 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
4604 MachinePointerInfo(SV));
4605}
4606
4607SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
4608 SelectionDAG &DAG) const {
4610 SystemZMachineFunctionInfo *FuncInfo =
4611 MF.getInfo<SystemZMachineFunctionInfo>();
4612 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4613
4614 SDValue Chain = Op.getOperand(0);
4615 SDValue Addr = Op.getOperand(1);
4616 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4617 SDLoc DL(Op);
4618
4619 // The initial values of each field.
4620 const unsigned NumFields = 4;
4621 SDValue Fields[NumFields] = {
4622 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
4623 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
4624 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),
4625 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)
4626 };
4627
4628 // Store each field into its respective slot.
4629 SDValue MemOps[NumFields];
4630 unsigned Offset = 0;
4631 for (unsigned I = 0; I < NumFields; ++I) {
4632 SDValue FieldAddr = Addr;
4633 if (Offset != 0)
4634 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
4636 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
4637 MachinePointerInfo(SV, Offset));
4638 Offset += 8;
4639 }
4640 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
4641}
4642
4643SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
4644 SelectionDAG &DAG) const {
4645 SDValue Chain = Op.getOperand(0);
4646 SDValue DstPtr = Op.getOperand(1);
4647 SDValue SrcPtr = Op.getOperand(2);
4648 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
4649 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
4650 SDLoc DL(Op);
4651
4652 uint32_t Sz =
4653 Subtarget.isTargetXPLINK64() ? DAG.getDataLayout().getPointerSize(0) : 32;
4654 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(Sz, DL),
4655 Align(8), Align(8), /*isVolatile*/ false,
4656 /*AlwaysInline*/ false,
4657 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
4658 MachinePointerInfo(SrcSV));
4659}
4660
4661SDValue
4662SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
4663 SelectionDAG &DAG) const {
4664 if (Subtarget.isTargetXPLINK64())
4665 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4666 else
4667 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4668}
4669
4670SDValue
4671SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
4672 SelectionDAG &DAG) const {
4673 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4675 bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");
4676 SDValue Chain = Op.getOperand(0);
4677 SDValue Size = Op.getOperand(1);
4678 SDValue Align = Op.getOperand(2);
4679 SDLoc DL(Op);
4680
4681 // If user has set the no alignment function attribute, ignore
4682 // alloca alignments.
4683 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4684
4685 uint64_t StackAlign = TFI->getStackAlignment();
4686 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
4687 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4688
4689 SDValue NeededSpace = Size;
4690
4691 // Add extra space for alignment if needed.
4692 EVT PtrVT = getPointerTy(MF.getDataLayout());
4693 if (ExtraAlignSpace)
4694 NeededSpace = DAG.getNode(ISD::ADD, DL, PtrVT, NeededSpace,
4695 DAG.getConstant(ExtraAlignSpace, DL, PtrVT));
4696
4697 bool IsSigned = false;
4698 bool DoesNotReturn = false;
4699 bool IsReturnValueUsed = false;
4700 EVT VT = Op.getValueType();
4701 SDValue AllocaCall =
4702 makeExternalCall(Chain, DAG, "@@ALCAXP", VT, ArrayRef(NeededSpace),
4703 CallingConv::C, IsSigned, DL, DoesNotReturn,
4704 IsReturnValueUsed)
4705 .first;
4706
4707 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
4708 // to end of call in order to ensure it isn't broken up from the call
4709 // sequence.
4710 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4711 Register SPReg = Regs.getStackPointerRegister();
4712 Chain = AllocaCall.getValue(1);
4713 SDValue Glue = AllocaCall.getValue(2);
4714 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, DL, SPReg, PtrVT, Glue);
4715 Chain = NewSPRegNode.getValue(1);
4716
4717 MVT PtrMVT = getPointerMemTy(MF.getDataLayout());
4718 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, PtrMVT);
4719 SDValue Result = DAG.getNode(ISD::ADD, DL, PtrMVT, NewSPRegNode, ArgAdjust);
4720
4721 // Dynamically realign if needed.
4722 if (ExtraAlignSpace) {
4723 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
4724 DAG.getConstant(ExtraAlignSpace, DL, PtrVT));
4725 Result = DAG.getNode(ISD::AND, DL, PtrVT, Result,
4726 DAG.getConstant(~(RequiredAlign - 1), DL, PtrVT));
4727 }
4728
4729 SDValue Ops[2] = {Result, Chain};
4730 return DAG.getMergeValues(Ops, DL);
4731}
4732
4733SDValue
4734SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4735 SelectionDAG &DAG) const {
4736 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4738 bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");
4739 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4740
4741 SDValue Chain = Op.getOperand(0);
4742 SDValue Size = Op.getOperand(1);
4743 SDValue Align = Op.getOperand(2);
4744 SDLoc DL(Op);
4745
4746 // If user has set the no alignment function attribute, ignore
4747 // alloca alignments.
4748 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4749
4750 uint64_t StackAlign = TFI->getStackAlignment();
4751 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
4752 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4753
4755 SDValue NeededSpace = Size;
4756
4757 // Get a reference to the stack pointer.
4758 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
4759
4760 // If we need a backchain, save it now.
4761 SDValue Backchain;
4762 if (StoreBackchain)
4763 Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
4764 MachinePointerInfo());
4765
4766 // Add extra space for alignment if needed.
4767 if (ExtraAlignSpace)
4768 NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace,
4769 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
4770
4771 // Get the new stack pointer value.
4772 SDValue NewSP;
4773 if (hasInlineStackProbe(MF)) {
4774 NewSP = DAG.getNode(SystemZISD::PROBED_ALLOCA, DL,
4775 DAG.getVTList(MVT::i64, MVT::Other), Chain, OldSP, NeededSpace);
4776 Chain = NewSP.getValue(1);
4777 }
4778 else {
4779 NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
4780 // Copy the new stack pointer back.
4781 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP);
4782 }
4783
4784 // The allocated data lives above the 160 bytes allocated for the standard
4785 // frame, plus any outgoing stack arguments. We don't know how much that
4786 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4787 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
4788 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
4789
4790 // Dynamically realign if needed.
4791 if (RequiredAlign > StackAlign) {
4792 Result =
4793 DAG.getNode(ISD::ADD, DL, MVT::i64, Result,
4794 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
4795 Result =
4796 DAG.getNode(ISD::AND, DL, MVT::i64, Result,
4797 DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));
4798 }
4799
4800 if (StoreBackchain)
4801 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
4802 MachinePointerInfo());
4803
4804 SDValue Ops[2] = { Result, Chain };
4805 return DAG.getMergeValues(Ops, DL);
4806}
4807
4808SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4809 SDValue Op, SelectionDAG &DAG) const {
4810 SDLoc DL(Op);
4811
4812 return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
4813}
4814
4815SDValue SystemZTargetLowering::lowerMULH(SDValue Op,
4816 SelectionDAG &DAG,
4817 unsigned Opcode) const {
4818 EVT VT = Op.getValueType();
4819 SDLoc DL(Op);
4820 SDValue Even, Odd;
4821
4822 // This custom expander is only used on z17 and later for 64-bit types.
4823 assert(!is32Bit(VT));
4824 assert(Subtarget.hasMiscellaneousExtensions2());
4825
4826 // SystemZISD::xMUL_LOHI returns the low result in the odd register and
4827 // the high result in the even register. Return the latter.
4828 lowerGR128Binary(DAG, DL, VT, Opcode,
4829 Op.getOperand(0), Op.getOperand(1), Even, Odd);
4830 return Even;
4831}
4832
4833SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4834 SelectionDAG &DAG) const {
4835 EVT VT = Op.getValueType();
4836 SDLoc DL(Op);
4837 SDValue Ops[2];
4838 if (is32Bit(VT))
4839 // Just do a normal 64-bit multiplication and extract the results.
4840 // We define this so that it can be used for constant division.
4841 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0),
4842 Op.getOperand(1), Ops[1], Ops[0]);
4843 else if (Subtarget.hasMiscellaneousExtensions2())
4844 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4845 // the high result in the even register. ISD::SMUL_LOHI is defined to
4846 // return the low half first, so the results are in reverse order.
4847 lowerGR128Binary(DAG, DL, VT, SystemZISD::SMUL_LOHI,
4848 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
4849 else {
4850 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4851 //
4852 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4853 //
4854 // but using the fact that the upper halves are either all zeros
4855 // or all ones:
4856 //
4857 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4858 //
4859 // and grouping the right terms together since they are quicker than the
4860 // multiplication:
4861 //
4862 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4863 SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
4864 SDValue LL = Op.getOperand(0);
4865 SDValue RL = Op.getOperand(1);
4866 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
4867 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
4868 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4869 // the high result in the even register. ISD::SMUL_LOHI is defined to
4870 // return the low half first, so the results are in reverse order.
4871 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
4872 LL, RL, Ops[1], Ops[0]);
4873 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
4874 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
4875 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
4876 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum);
4877 }
4878 return DAG.getMergeValues(Ops, DL);
4879}
4880
4881SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4882 SelectionDAG &DAG) const {
4883 EVT VT = Op.getValueType();
4884 SDLoc DL(Op);
4885 SDValue Ops[2];
4886 if (is32Bit(VT))
4887 // Just do a normal 64-bit multiplication and extract the results.
4888 // We define this so that it can be used for constant division.
4889 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0),
4890 Op.getOperand(1), Ops[1], Ops[0]);
4891 else
4892 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4893 // the high result in the even register. ISD::UMUL_LOHI is defined to
4894 // return the low half first, so the results are in reverse order.
4895 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
4896 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
4897 return DAG.getMergeValues(Ops, DL);
4898}
4899
4900SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4901 SelectionDAG &DAG) const {
4902 SDValue Op0 = Op.getOperand(0);
4903 SDValue Op1 = Op.getOperand(1);
4904 EVT VT = Op.getValueType();
4905 SDLoc DL(Op);
4906
4907 // We use DSGF for 32-bit division. This means the first operand must
4908 // always be 64-bit, and the second operand should be 32-bit whenever
4909 // that is possible, to improve performance.
4910 if (is32Bit(VT))
4911 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0);
4912 else if (DAG.ComputeNumSignBits(Op1) > 32)
4913 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
4914
4915 // DSG(F) returns the remainder in the even register and the
4916 // quotient in the odd register.
4917 SDValue Ops[2];
4918 lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]);
4919 return DAG.getMergeValues(Ops, DL);
4920}
4921
4922SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4923 SelectionDAG &DAG) const {
4924 EVT VT = Op.getValueType();
4925 SDLoc DL(Op);
4926
4927 // DL(G) returns the remainder in the even register and the
4928 // quotient in the odd register.
4929 SDValue Ops[2];
4930 lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM,
4931 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
4932 return DAG.getMergeValues(Ops, DL);
4933}
4934
4935SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4936 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4937
4938 // Get the known-zero masks for each operand.
4939 SDValue Ops[] = {Op.getOperand(0), Op.getOperand(1)};
4940 KnownBits Known[2] = {DAG.computeKnownBits(Ops[0]),
4941 DAG.computeKnownBits(Ops[1])};
4942
4943 // See if the upper 32 bits of one operand and the lower 32 bits of the
4944 // other are known zero. They are the low and high operands respectively.
4945 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4946 Known[1].Zero.getZExtValue() };
4947 unsigned High, Low;
4948 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4949 High = 1, Low = 0;
4950 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4951 High = 0, Low = 1;
4952 else
4953 return Op;
4954
4955 SDValue LowOp = Ops[Low];
4956 SDValue HighOp = Ops[High];
4957
4958 // If the high part is a constant, we're better off using IILH.
4959 if (HighOp.getOpcode() == ISD::Constant)
4960 return Op;
4961
4962 // If the low part is a constant that is outside the range of LHI,
4963 // then we're better off using IILF.
4964 if (LowOp.getOpcode() == ISD::Constant) {
4965 int64_t Value = int32_t(LowOp->getAsZExtVal());
4966 if (!isInt<16>(Value))
4967 return Op;
4968 }
4969
4970 // Check whether the high part is an AND that doesn't change the
4971 // high 32 bits and just masks out low bits. We can skip it if so.
4972 if (HighOp.getOpcode() == ISD::AND &&
4973 HighOp.getOperand(1).getOpcode() == ISD::Constant) {
4974 SDValue HighOp0 = HighOp.getOperand(0);
4976 if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff))))
4977 HighOp = HighOp0;
4978 }
4979
4980 // Take advantage of the fact that all GR32 operations only change the
4981 // low 32 bits by truncating Low to an i32 and inserting it directly
4982 // using a subreg. The interesting cases are those where the truncation
4983 // can be folded.
4984 SDLoc DL(Op);
4985 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
4986 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
4987 MVT::i64, HighOp, Low32);
4988}
4989
4990// Lower SADDO/SSUBO/UADDO/USUBO nodes.
4991SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4992 SelectionDAG &DAG) const {
4993 SDNode *N = Op.getNode();
4994 SDValue LHS = N->getOperand(0);
4995 SDValue RHS = N->getOperand(1);
4996 SDLoc DL(N);
4997
4998 if (N->getValueType(0) == MVT::i128) {
4999 unsigned BaseOp = 0;
5000 unsigned FlagOp = 0;
5001 bool IsBorrow = false;
5002 switch (Op.getOpcode()) {
5003 default: llvm_unreachable("Unknown instruction!");
5004 case ISD::UADDO:
5005 BaseOp = ISD::ADD;
5006 FlagOp = SystemZISD::VACC;
5007 break;
5008 case ISD::USUBO:
5009 BaseOp = ISD::SUB;
5010 FlagOp = SystemZISD::VSCBI;
5011 IsBorrow = true;
5012 break;
5013 }
5014 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS);
5015 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS);
5016 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag,
5017 DAG.getValueType(MVT::i1));
5018 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));
5019 if (IsBorrow)
5020 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(),
5021 Flag, DAG.getConstant(1, DL, Flag.getValueType()));
5022 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);
5023 }
5024
5025 unsigned BaseOp = 0;
5026 unsigned CCValid = 0;
5027 unsigned CCMask = 0;
5028
5029 switch (Op.getOpcode()) {
5030 default: llvm_unreachable("Unknown instruction!");
5031 case ISD::SADDO:
5032 BaseOp = SystemZISD::SADDO;
5033 CCValid = SystemZ::CCMASK_ARITH;
5035 break;
5036 case ISD::SSUBO:
5037 BaseOp = SystemZISD::SSUBO;
5038 CCValid = SystemZ::CCMASK_ARITH;
5040 break;
5041 case ISD::UADDO:
5042 BaseOp = SystemZISD::UADDO;
5043 CCValid = SystemZ::CCMASK_LOGICAL;
5045 break;
5046 case ISD::USUBO:
5047 BaseOp = SystemZISD::USUBO;
5048 CCValid = SystemZ::CCMASK_LOGICAL;
5050 break;
5051 }
5052
5053 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
5054 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
5055
5056 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
5057 if (N->getValueType(1) == MVT::i1)
5058 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
5059
5060 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
5061}
5062
5063static bool isAddCarryChain(SDValue Carry) {
5064 while (Carry.getOpcode() == ISD::UADDO_CARRY &&
5065 Carry->getValueType(0) != MVT::i128)
5066 Carry = Carry.getOperand(2);
5067 return Carry.getOpcode() == ISD::UADDO &&
5068 Carry->getValueType(0) != MVT::i128;
5069}
5070
5071static bool isSubBorrowChain(SDValue Carry) {
5072 while (Carry.getOpcode() == ISD::USUBO_CARRY &&
5073 Carry->getValueType(0) != MVT::i128)
5074 Carry = Carry.getOperand(2);
5075 return Carry.getOpcode() == ISD::USUBO &&
5076 Carry->getValueType(0) != MVT::i128;
5077}
5078
5079// Lower UADDO_CARRY/USUBO_CARRY nodes.
5080SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
5081 SelectionDAG &DAG) const {
5082
5083 SDNode *N = Op.getNode();
5084 MVT VT = N->getSimpleValueType(0);
5085
5086 // Let legalize expand this if it isn't a legal type yet.
5087 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
5088 return SDValue();
5089
5090 SDValue LHS = N->getOperand(0);
5091 SDValue RHS = N->getOperand(1);
5092 SDValue Carry = Op.getOperand(2);
5093 SDLoc DL(N);
5094
5095 if (VT == MVT::i128) {
5096 unsigned BaseOp = 0;
5097 unsigned FlagOp = 0;
5098 bool IsBorrow = false;
5099 switch (Op.getOpcode()) {
5100 default: llvm_unreachable("Unknown instruction!");
5101 case ISD::UADDO_CARRY:
5102 BaseOp = SystemZISD::VAC;
5103 FlagOp = SystemZISD::VACCC;
5104 break;
5105 case ISD::USUBO_CARRY:
5106 BaseOp = SystemZISD::VSBI;
5107 FlagOp = SystemZISD::VSBCBI;
5108 IsBorrow = true;
5109 break;
5110 }
5111 if (IsBorrow)
5112 Carry = DAG.getNode(ISD::XOR, DL, Carry.getValueType(),
5113 Carry, DAG.getConstant(1, DL, Carry.getValueType()));
5114 Carry = DAG.getZExtOrTrunc(Carry, DL, MVT::i128);
5115 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS, Carry);
5116 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS, Carry);
5117 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag,
5118 DAG.getValueType(MVT::i1));
5119 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));
5120 if (IsBorrow)
5121 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(),
5122 Flag, DAG.getConstant(1, DL, Flag.getValueType()));
5123 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);
5124 }
5125
5126 unsigned BaseOp = 0;
5127 unsigned CCValid = 0;
5128 unsigned CCMask = 0;
5129
5130 switch (Op.getOpcode()) {
5131 default: llvm_unreachable("Unknown instruction!");
5132 case ISD::UADDO_CARRY:
5133 if (!isAddCarryChain(Carry))
5134 return SDValue();
5135
5136 BaseOp = SystemZISD::ADDCARRY;
5137 CCValid = SystemZ::CCMASK_LOGICAL;
5139 break;
5140 case ISD::USUBO_CARRY:
5141 if (!isSubBorrowChain(Carry))
5142 return SDValue();
5143
5144 BaseOp = SystemZISD::SUBCARRY;
5145 CCValid = SystemZ::CCMASK_LOGICAL;
5147 break;
5148 }
5149
5150 // Set the condition code from the carry flag.
5151 Carry = DAG.getNode(SystemZISD::GET_CCMASK, DL, MVT::i32, Carry,
5152 DAG.getConstant(CCValid, DL, MVT::i32),
5153 DAG.getConstant(CCMask, DL, MVT::i32));
5154
5155 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
5156 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry);
5157
5158 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
5159 if (N->getValueType(1) == MVT::i1)
5160 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
5161
5162 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
5163}
5164
5165SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
5166 SelectionDAG &DAG) const {
5167 EVT VT = Op.getValueType();
5168 SDLoc DL(Op);
5169 Op = Op.getOperand(0);
5170
5171 if (VT.getScalarSizeInBits() == 128) {
5172 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op);
5173 Op = DAG.getNode(ISD::CTPOP, DL, MVT::v2i64, Op);
5174 SDValue Tmp = DAG.getSplatBuildVector(MVT::v2i64, DL,
5175 DAG.getConstant(0, DL, MVT::i64));
5176 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
5177 return Op;
5178 }
5179
5180 // Handle vector types via VPOPCT.
5181 if (VT.isVector()) {
5182 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
5183 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
5184 switch (VT.getScalarSizeInBits()) {
5185 case 8:
5186 break;
5187 case 16: {
5188 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
5189 SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
5190 SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
5191 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
5192 Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
5193 break;
5194 }
5195 case 32: {
5196 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
5197 DAG.getConstant(0, DL, MVT::i32));
5198 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
5199 break;
5200 }
5201 case 64: {
5202 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
5203 DAG.getConstant(0, DL, MVT::i32));
5204 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
5205 Tmp = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, Tmp);
5206 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
5207 break;
5208 }
5209 default:
5210 llvm_unreachable("Unexpected type");
5211 }
5212 return Op;
5213 }
5214
5215 // Get the known-zero mask for the operand.
5216 KnownBits Known = DAG.computeKnownBits(Op);
5217 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
5218 if (NumSignificantBits == 0)
5219 return DAG.getConstant(0, DL, VT);
5220
5221 // Skip known-zero high parts of the operand.
5222 int64_t OrigBitSize = VT.getSizeInBits();
5223 int64_t BitSize = llvm::bit_ceil(NumSignificantBits);
5224 BitSize = std::min(BitSize, OrigBitSize);
5225
5226 // The POPCNT instruction counts the number of bits in each byte.
5227 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op);
5228 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op);
5229 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
5230
5231 // Add up per-byte counts in a binary tree. All bits of Op at
5232 // position larger than BitSize remain zero throughout.
5233 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
5234 SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
5235 if (BitSize != OrigBitSize)
5236 Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
5237 DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
5238 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
5239 }
5240
5241 // Extract overall result from high byte.
5242 if (BitSize > 8)
5243 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
5244 DAG.getConstant(BitSize - 8, DL, VT));
5245
5246 return Op;
5247}
5248
5249SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
5250 SelectionDAG &DAG) const {
5251 SDLoc DL(Op);
5252 AtomicOrdering FenceOrdering =
5253 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
5254 SyncScope::ID FenceSSID =
5255 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
5256
5257 // The only fence that needs an instruction is a sequentially-consistent
5258 // cross-thread fence.
5259 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
5260 FenceSSID == SyncScope::System) {
5261 return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
5262 Op.getOperand(0)),
5263 0);
5264 }
5265
5266 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5267 return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
5268}
5269
5270SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
5271 SelectionDAG &DAG) const {
5272 EVT RegVT = Op.getValueType();
5273 if (RegVT.getSizeInBits() == 128)
5274 return lowerATOMIC_LDST_I128(Op, DAG);
5275 return lowerLoadF16(Op, DAG);
5276}
5277
5278SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
5279 SelectionDAG &DAG) const {
5280 auto *Node = cast<AtomicSDNode>(Op.getNode());
5281 if (Node->getMemoryVT().getSizeInBits() == 128)
5282 return lowerATOMIC_LDST_I128(Op, DAG);
5283 return lowerStoreF16(Op, DAG);
5284}
5285
5286SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
5287 SelectionDAG &DAG) const {
5288 auto *Node = cast<AtomicSDNode>(Op.getNode());
5289 assert(
5290 (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&
5291 "Only custom lowering i128 or f128.");
5292 // Use same code to handle both legal and non-legal i128 types.
5294 LowerOperationWrapper(Node, Results, DAG);
5295 return DAG.getMergeValues(Results, SDLoc(Op));
5296}
5297
5298// Prepare for a Compare And Swap for a subword operation. This needs to be
5299// done in memory with 4 bytes at natural alignment.
5301 SDValue &AlignedAddr, SDValue &BitShift,
5302 SDValue &NegBitShift) {
5303 EVT PtrVT = Addr.getValueType();
5304 EVT WideVT = MVT::i32;
5305
5306 // Get the address of the containing word.
5307 AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
5308 DAG.getSignedConstant(-4, DL, PtrVT));
5309
5310 // Get the number of bits that the word must be rotated left in order
5311 // to bring the field to the top bits of a GR32.
5312 BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
5313 DAG.getConstant(3, DL, PtrVT));
5314 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
5315
5316 // Get the complementing shift amount, for rotating a field in the top
5317 // bits back to its proper position.
5318 NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
5319 DAG.getConstant(0, DL, WideVT), BitShift);
5320
5321}
5322
5323// N is an 8- or 16-bit ATOMIC_LOAD_* operation. Lower it into the fullword
5324// ATOMIC_LOADW_* operation.
5325void SystemZTargetLowering::lowerATOMIC_LOAD_OP(
5327 unsigned Opcode) const {
5328 auto *Node = cast<AtomicSDNode>(N);
5329
5330 EVT NarrowVT = Node->getMemoryVT();
5331 EVT WideVT = MVT::i32;
5332 int64_t BitSize = NarrowVT.getSizeInBits();
5333 SDValue ChainIn = Node->getChain();
5334 SDValue Addr = Node->getBasePtr();
5335 SDValue Src2 = Node->getVal();
5336 MachineMemOperand *MMO = Node->getMemOperand();
5337 SDLoc DL(Node);
5338
5339 // Convert atomic subtracts of constants into additions.
5340 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5341 if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
5342 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5343 Src2 = DAG.getConstant(-Const->getAPIntValue(), DL, Src2.getValueType());
5344 }
5345
5346 SDValue AlignedAddr, BitShift, NegBitShift;
5347 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5348
5349 // Extend the source operand to 32 bits and prepare it for the inner loop.
5350 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
5351 // operations require the source to be shifted in advance. (This shift
5352 // can be folded if the source is constant.) For AND and NAND, the lower
5353 // bits must be set, while for other opcodes they should be left clear.
5354 Src2 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Src2);
5355 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5356 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
5357 DAG.getConstant(32 - BitSize, DL, WideVT));
5358 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5359 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5360 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
5361 DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
5362
5363 // Construct the ATOMIC_LOADW_* node.
5364 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
5365 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
5366 DAG.getConstant(BitSize, DL, WideVT) };
5367 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
5368 NarrowVT, MMO);
5369
5370 // Rotate the result of the final CS so that the field is in the lower
5371 // bits of a GR32, then truncate it.
5372 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
5373 DAG.getConstant(BitSize, DL, WideVT));
5374 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
5375 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Result));
5376 Results.push_back(AtomicOp.getValue(1));
5377}
5378
5379// Op is a 32- or 64-bit ATOMIC_LOAD_SUB operation. Convert it into an
5380// addition.
5381SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
5382 SelectionDAG &DAG) const {
5383 auto *Node = cast<AtomicSDNode>(Op.getNode());
5384 EVT MemVT = Node->getMemoryVT();
5385 assert(Op.getValueType() == MemVT && "Mismatched VTs");
5386 assert(Subtarget.hasInterlockedAccess1() &&
5387 "Should have been expanded by AtomicExpand pass.");
5388 SDValue Src2 = Node->getVal();
5389 SDLoc DL(Src2);
5390 SDValue NegSrc2 =
5391 DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2);
5392 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, Node->getChain(),
5393 Node->getBasePtr(), NegSrc2, Node->getMemOperand());
5394}
5395
5396// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
5397SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
5398 SelectionDAG &DAG) const {
5399 auto *Node = cast<AtomicSDNode>(Op.getNode());
5400 SDValue ChainIn = Node->getOperand(0);
5401 SDValue Addr = Node->getOperand(1);
5402 SDValue CmpVal = Node->getOperand(2);
5403 SDValue SwapVal = Node->getOperand(3);
5404 MachineMemOperand *MMO = Node->getMemOperand();
5405 SDLoc DL(Node);
5406
5407 if (Node->getMemoryVT() == MVT::i128) {
5408 // Use same code to handle both legal and non-legal i128 types.
5410 LowerOperationWrapper(Node, Results, DAG);
5411 return DAG.getMergeValues(Results, DL);
5412 }
5413
5414 // We have native support for 32-bit and 64-bit compare and swap, but we
5415 // still need to expand extracting the "success" result from the CC.
5416 EVT NarrowVT = Node->getMemoryVT();
5417 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5418 if (NarrowVT == WideVT) {
5419 SDVTList Tys = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
5420 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5421 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP,
5422 DL, Tys, Ops, NarrowVT, MMO);
5423 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
5425
5426 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
5427 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
5428 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
5429 return SDValue();
5430 }
5431
5432 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
5433 // via a fullword ATOMIC_CMP_SWAPW operation.
5434 int64_t BitSize = NarrowVT.getSizeInBits();
5435
5436 SDValue AlignedAddr, BitShift, NegBitShift;
5437 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5438
5439 // Construct the ATOMIC_CMP_SWAPW node.
5440 SDVTList VTList = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
5441 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5442 NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
5443 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
5444 VTList, Ops, NarrowVT, MMO);
5445 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
5447
5448 // emitAtomicCmpSwapW() will zero extend the result (original value).
5449 SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
5450 DAG.getValueType(NarrowVT));
5451 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
5452 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
5453 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
5454 return SDValue();
5455}
5456
5458SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
5459 // Because of how we convert atomic_load and atomic_store to normal loads and
5460 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5461 // since DAGCombine hasn't been updated to account for atomic, but non
5462 // volatile loads. (See D57601)
5463 if (auto *SI = dyn_cast<StoreInst>(&I))
5464 if (SI->isAtomic())
5466 if (auto *LI = dyn_cast<LoadInst>(&I))
5467 if (LI->isAtomic())
5469 if (auto *AI = dyn_cast<AtomicRMWInst>(&I))
5470 if (AI->isAtomic())
5472 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(&I))
5473 if (AI->isAtomic())
5476}
5477
5478SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
5479 SelectionDAG &DAG) const {
5481 auto *Regs = Subtarget.getSpecialRegisters();
5483 report_fatal_error("Variable-sized stack allocations are not supported "
5484 "in GHC calling convention");
5485 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op),
5486 Regs->getStackPointerRegister(), Op.getValueType());
5487}
5488
5489SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
5490 SelectionDAG &DAG) const {
5492 auto *Regs = Subtarget.getSpecialRegisters();
5493
5495 report_fatal_error("Variable-sized stack allocations are not supported "
5496 "in GHC calling convention");
5497
5498 SDValue Chain = Op.getOperand(0);
5499 SDValue NewSP = Op.getOperand(1);
5500 SDLoc DL(Op);
5501
5502 // XPLINK: @@ALCAXP (lowerDYNAMIC_STACKALLOC_XPLINK) moves the frame header
5503 // (register save area and reserved words at SP + bias) down with the stack
5504 // pointer, and calls made while the stack pointer is lowered place their
5505 // arguments over the old header. The current header must be copied back to
5506 // the restored stack pointer, otherwise the epilogue and the LE stack walk
5507 // use a clobbered header (abend U4083). No signal may be delivered while
5508 // the stack pointer is changed and the header is copied, so this is done in
5509 // a stub routine (emitted once per module by the AsmPrinter), which gets
5510 // the new stack pointer in R1 and is called with BASR 3,3.
5511 if (Subtarget.isTargetXPLINK64()) {
5512 SDValue Glue;
5513 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, NewSP, Glue);
5514 Glue = Chain.getValue(1);
5515 return DAG.getNode(SystemZISD::XPLINK_STACKRESTORE, DL, MVT::Other, Chain,
5516 Glue);
5517 }
5518
5519 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
5520 SDValue Backchain;
5521 if (StoreBackchain) {
5522 SDValue OldSP = DAG.getCopyFromReg(
5523 Chain, DL, Regs->getStackPointerRegister(), MVT::i64);
5524 Backchain =
5525 DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
5526 MachinePointerInfo());
5527 }
5528
5529 Chain = DAG.getCopyToReg(Chain, DL, Regs->getStackPointerRegister(), NewSP);
5530
5531 if (StoreBackchain)
5532 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
5533 MachinePointerInfo());
5534
5535 return Chain;
5536}
5537
5538SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
5539 SelectionDAG &DAG) const {
5540 bool IsData = Op.getConstantOperandVal(4);
5541 if (!IsData)
5542 // Just preserve the chain.
5543 return Op.getOperand(0);
5544
5545 SDLoc DL(Op);
5546 bool IsWrite = Op.getConstantOperandVal(2);
5547 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
5548 auto *Node = cast<MemIntrinsicSDNode>(Op.getNode());
5549 SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32),
5550 Op.getOperand(1)};
5551 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
5552 Node->getVTList(), Ops,
5553 Node->getMemoryVT(), Node->getMemOperand());
5554}
5555
5556SDValue
5557SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5558 SelectionDAG &DAG) const {
5559 unsigned Opcode, CCValid;
5560 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
5561 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
5562 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5563 SDValue CC = getCCResult(DAG, SDValue(Node, 0));
5564 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), CC);
5565 return SDValue();
5566 }
5567
5568 return SDValue();
5569}
5570
5571SDValue
5572SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5573 SelectionDAG &DAG) const {
5574 unsigned Opcode, CCValid;
5575 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
5576 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5577 if (Op->getNumValues() == 1)
5578 return getCCResult(DAG, SDValue(Node, 0));
5579 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
5580 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(),
5581 SDValue(Node, 0), getCCResult(DAG, SDValue(Node, 1)));
5582 }
5583
5584 unsigned Id = Op.getConstantOperandVal(0);
5585 switch (Id) {
5586 case Intrinsic::thread_pointer:
5587 return lowerThreadPointer(SDLoc(Op), DAG);
5588
5589 case Intrinsic::s390_vpdi:
5590 return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(),
5591 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5592
5593 case Intrinsic::s390_vperm:
5594 return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(),
5595 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5596
5597 case Intrinsic::s390_vuphb:
5598 case Intrinsic::s390_vuphh:
5599 case Intrinsic::s390_vuphf:
5600 case Intrinsic::s390_vuphg:
5601 return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(),
5602 Op.getOperand(1));
5603
5604 case Intrinsic::s390_vuplhb:
5605 case Intrinsic::s390_vuplhh:
5606 case Intrinsic::s390_vuplhf:
5607 case Intrinsic::s390_vuplhg:
5608 return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(),
5609 Op.getOperand(1));
5610
5611 case Intrinsic::s390_vuplb:
5612 case Intrinsic::s390_vuplhw:
5613 case Intrinsic::s390_vuplf:
5614 case Intrinsic::s390_vuplg:
5615 return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(),
5616 Op.getOperand(1));
5617
5618 case Intrinsic::s390_vupllb:
5619 case Intrinsic::s390_vupllh:
5620 case Intrinsic::s390_vupllf:
5621 case Intrinsic::s390_vupllg:
5622 return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(),
5623 Op.getOperand(1));
5624
5625 case Intrinsic::s390_vsumb:
5626 case Intrinsic::s390_vsumh:
5627 case Intrinsic::s390_vsumgh:
5628 case Intrinsic::s390_vsumgf:
5629 case Intrinsic::s390_vsumqf:
5630 case Intrinsic::s390_vsumqg:
5631 return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(),
5632 Op.getOperand(1), Op.getOperand(2));
5633
5634 case Intrinsic::s390_vaq:
5635 return DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(),
5636 Op.getOperand(1), Op.getOperand(2));
5637 case Intrinsic::s390_vaccb:
5638 case Intrinsic::s390_vacch:
5639 case Intrinsic::s390_vaccf:
5640 case Intrinsic::s390_vaccg:
5641 case Intrinsic::s390_vaccq:
5642 return DAG.getNode(SystemZISD::VACC, SDLoc(Op), Op.getValueType(),
5643 Op.getOperand(1), Op.getOperand(2));
5644 case Intrinsic::s390_vacq:
5645 return DAG.getNode(SystemZISD::VAC, SDLoc(Op), Op.getValueType(),
5646 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5647 case Intrinsic::s390_vacccq:
5648 return DAG.getNode(SystemZISD::VACCC, SDLoc(Op), Op.getValueType(),
5649 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5650
5651 case Intrinsic::s390_vsq:
5652 return DAG.getNode(ISD::SUB, SDLoc(Op), Op.getValueType(),
5653 Op.getOperand(1), Op.getOperand(2));
5654 case Intrinsic::s390_vscbib:
5655 case Intrinsic::s390_vscbih:
5656 case Intrinsic::s390_vscbif:
5657 case Intrinsic::s390_vscbig:
5658 case Intrinsic::s390_vscbiq:
5659 return DAG.getNode(SystemZISD::VSCBI, SDLoc(Op), Op.getValueType(),
5660 Op.getOperand(1), Op.getOperand(2));
5661 case Intrinsic::s390_vsbiq:
5662 return DAG.getNode(SystemZISD::VSBI, SDLoc(Op), Op.getValueType(),
5663 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5664 case Intrinsic::s390_vsbcbiq:
5665 return DAG.getNode(SystemZISD::VSBCBI, SDLoc(Op), Op.getValueType(),
5666 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5667
5668 case Intrinsic::s390_vmhb:
5669 case Intrinsic::s390_vmhh:
5670 case Intrinsic::s390_vmhf:
5671 case Intrinsic::s390_vmhg:
5672 case Intrinsic::s390_vmhq:
5673 return DAG.getNode(ISD::MULHS, SDLoc(Op), Op.getValueType(),
5674 Op.getOperand(1), Op.getOperand(2));
5675 case Intrinsic::s390_vmlhb:
5676 case Intrinsic::s390_vmlhh:
5677 case Intrinsic::s390_vmlhf:
5678 case Intrinsic::s390_vmlhg:
5679 case Intrinsic::s390_vmlhq:
5680 return DAG.getNode(ISD::MULHU, SDLoc(Op), Op.getValueType(),
5681 Op.getOperand(1), Op.getOperand(2));
5682
5683 case Intrinsic::s390_vmahb:
5684 case Intrinsic::s390_vmahh:
5685 case Intrinsic::s390_vmahf:
5686 case Intrinsic::s390_vmahg:
5687 case Intrinsic::s390_vmahq:
5688 return DAG.getNode(SystemZISD::VMAH, SDLoc(Op), Op.getValueType(),
5689 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5690 case Intrinsic::s390_vmalhb:
5691 case Intrinsic::s390_vmalhh:
5692 case Intrinsic::s390_vmalhf:
5693 case Intrinsic::s390_vmalhg:
5694 case Intrinsic::s390_vmalhq:
5695 return DAG.getNode(SystemZISD::VMALH, SDLoc(Op), Op.getValueType(),
5696 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
5697
5698 case Intrinsic::s390_vmeb:
5699 case Intrinsic::s390_vmeh:
5700 case Intrinsic::s390_vmef:
5701 case Intrinsic::s390_vmeg:
5702 return DAG.getNode(SystemZISD::VME, SDLoc(Op), Op.getValueType(),
5703 Op.getOperand(1), Op.getOperand(2));
5704 case Intrinsic::s390_vmleb:
5705 case Intrinsic::s390_vmleh:
5706 case Intrinsic::s390_vmlef:
5707 case Intrinsic::s390_vmleg:
5708 return DAG.getNode(SystemZISD::VMLE, SDLoc(Op), Op.getValueType(),
5709 Op.getOperand(1), Op.getOperand(2));
5710 case Intrinsic::s390_vmob:
5711 case Intrinsic::s390_vmoh:
5712 case Intrinsic::s390_vmof:
5713 case Intrinsic::s390_vmog:
5714 return DAG.getNode(SystemZISD::VMO, SDLoc(Op), Op.getValueType(),
5715 Op.getOperand(1), Op.getOperand(2));
5716 case Intrinsic::s390_vmlob:
5717 case Intrinsic::s390_vmloh:
5718 case Intrinsic::s390_vmlof:
5719 case Intrinsic::s390_vmlog:
5720 return DAG.getNode(SystemZISD::VMLO, SDLoc(Op), Op.getValueType(),
5721 Op.getOperand(1), Op.getOperand(2));
5722
5723 case Intrinsic::s390_vmaeb:
5724 case Intrinsic::s390_vmaeh:
5725 case Intrinsic::s390_vmaef:
5726 case Intrinsic::s390_vmaeg:
5727 return DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(),
5728 DAG.getNode(SystemZISD::VME, SDLoc(Op), Op.getValueType(),
5729 Op.getOperand(1), Op.getOperand(2)),
5730 Op.getOperand(3));
5731 case Intrinsic::s390_vmaleb:
5732 case Intrinsic::s390_vmaleh:
5733 case Intrinsic::s390_vmalef:
5734 case Intrinsic::s390_vmaleg:
5735 return DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(),
5736 DAG.getNode(SystemZISD::VMLE, SDLoc(Op), Op.getValueType(),
5737 Op.getOperand(1), Op.getOperand(2)),
5738 Op.getOperand(3));
5739 case Intrinsic::s390_vmaob:
5740 case Intrinsic::s390_vmaoh:
5741 case Intrinsic::s390_vmaof:
5742 case Intrinsic::s390_vmaog:
5743 return DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(),
5744 DAG.getNode(SystemZISD::VMO, SDLoc(Op), Op.getValueType(),
5745 Op.getOperand(1), Op.getOperand(2)),
5746 Op.getOperand(3));
5747 case Intrinsic::s390_vmalob:
5748 case Intrinsic::s390_vmaloh:
5749 case Intrinsic::s390_vmalof:
5750 case Intrinsic::s390_vmalog:
5751 return DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(),
5752 DAG.getNode(SystemZISD::VMLO, SDLoc(Op), Op.getValueType(),
5753 Op.getOperand(1), Op.getOperand(2)),
5754 Op.getOperand(3));
5755 }
5756
5757 return SDValue();
5758}
5759
5760namespace {
5761// Says that SystemZISD operation Opcode can be used to perform the equivalent
5762// of a VPERM with permute vector Bytes. If Opcode takes three operands,
5763// Operand is the constant third operand, otherwise it is the number of
5764// bytes in each element of the result.
5765struct Permute {
5766 unsigned Opcode;
5767 unsigned Operand;
5768 unsigned char Bytes[SystemZ::VectorBytes];
5769};
5770}
5771
5772static const Permute PermuteForms[] = {
5773 // VMRHG
5774 { SystemZISD::MERGE_HIGH, 8,
5775 { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5776 // VMRHF
5777 { SystemZISD::MERGE_HIGH, 4,
5778 { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5779 // VMRHH
5780 { SystemZISD::MERGE_HIGH, 2,
5781 { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5782 // VMRHB
5783 { SystemZISD::MERGE_HIGH, 1,
5784 { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5785 // VMRLG
5786 { SystemZISD::MERGE_LOW, 8,
5787 { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5788 // VMRLF
5789 { SystemZISD::MERGE_LOW, 4,
5790 { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5791 // VMRLH
5792 { SystemZISD::MERGE_LOW, 2,
5793 { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5794 // VMRLB
5795 { SystemZISD::MERGE_LOW, 1,
5796 { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5797 // VPKG
5798 { SystemZISD::PACK, 4,
5799 { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5800 // VPKF
5801 { SystemZISD::PACK, 2,
5802 { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5803 // VPKH
5804 { SystemZISD::PACK, 1,
5805 { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5806 // VPDI V1, V2, 4 (low half of V1, high half of V2)
5807 { SystemZISD::PERMUTE_DWORDS, 4,
5808 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5809 // VPDI V1, V2, 1 (high half of V1, low half of V2)
5810 { SystemZISD::PERMUTE_DWORDS, 1,
5811 { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5812};
5813
5814// Called after matching a vector shuffle against a particular pattern.
5815// Both the original shuffle and the pattern have two vector operands.
5816// OpNos[0] is the operand of the original shuffle that should be used for
5817// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
5818// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
5819// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
5820// for operands 0 and 1 of the pattern.
5821static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
5822 if (OpNos[0] < 0) {
5823 if (OpNos[1] < 0)
5824 return false;
5825 OpNo0 = OpNo1 = OpNos[1];
5826 } else if (OpNos[1] < 0) {
5827 OpNo0 = OpNo1 = OpNos[0];
5828 } else {
5829 OpNo0 = OpNos[0];
5830 OpNo1 = OpNos[1];
5831 }
5832 return true;
5833}
5834
5835// Bytes is a VPERM-like permute vector, except that -1 is used for
5836// undefined bytes. Return true if the VPERM can be implemented using P.
5837// When returning true set OpNo0 to the VPERM operand that should be
5838// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5839//
5840// For example, if swapping the VPERM operands allows P to match, OpNo0
5841// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5842// operand, but rewriting it to use two duplicated operands allows it to
5843// match P, then OpNo0 and OpNo1 will be the same.
5844static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5845 unsigned &OpNo0, unsigned &OpNo1) {
5846 int OpNos[] = { -1, -1 };
5847 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5848 int Elt = Bytes[I];
5849 if (Elt >= 0) {
5850 // Make sure that the two permute vectors use the same suboperand
5851 // byte number. Only the operand numbers (the high bits) are
5852 // allowed to differ.
5853 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5854 return false;
5855 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5856 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5857 // Make sure that the operand mappings are consistent with previous
5858 // elements.
5859 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5860 return false;
5861 OpNos[ModelOpNo] = RealOpNo;
5862 }
5863 }
5864 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5865}
5866
5867// As above, but search for a matching permute.
5868static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5869 unsigned &OpNo0, unsigned &OpNo1) {
5870 for (auto &P : PermuteForms)
5871 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5872 return &P;
5873 return nullptr;
5874}
5875
5876// Bytes is a VPERM-like permute vector, except that -1 is used for
5877// undefined bytes. This permute is an operand of an outer permute.
5878// See whether redistributing the -1 bytes gives a shuffle that can be
5879// implemented using P. If so, set Transform to a VPERM-like permute vector
5880// that, when applied to the result of P, gives the original permute in Bytes.
5882 const Permute &P,
5883 SmallVectorImpl<int> &Transform) {
5884 unsigned To = 0;
5885 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5886 int Elt = Bytes[From];
5887 if (Elt < 0)
5888 // Byte number From of the result is undefined.
5889 Transform[From] = -1;
5890 else {
5891 while (P.Bytes[To] != Elt) {
5892 To += 1;
5893 if (To == SystemZ::VectorBytes)
5894 return false;
5895 }
5896 Transform[From] = To;
5897 }
5898 }
5899 return true;
5900}
5901
5902// As above, but search for a matching permute.
5903static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5904 SmallVectorImpl<int> &Transform) {
5905 for (auto &P : PermuteForms)
5906 if (matchDoublePermute(Bytes, P, Transform))
5907 return &P;
5908 return nullptr;
5909}
5910
5911// Convert the mask of the given shuffle op into a byte-level mask,
5912// as if it had type vNi8.
5913static bool getVPermMask(SDValue ShuffleOp,
5914 SmallVectorImpl<int> &Bytes) {
5915 EVT VT = ShuffleOp.getValueType();
5916 unsigned NumElements = VT.getVectorNumElements();
5917 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5918
5919 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(ShuffleOp)) {
5920 Bytes.resize(NumElements * BytesPerElement, -1);
5921 for (unsigned I = 0; I < NumElements; ++I) {
5922 int Index = VSN->getMaskElt(I);
5923 if (Index >= 0)
5924 for (unsigned J = 0; J < BytesPerElement; ++J)
5925 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5926 }
5927 return true;
5928 }
5929 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5930 isa<ConstantSDNode>(ShuffleOp.getOperand(1))) {
5931 unsigned Index = ShuffleOp.getConstantOperandVal(1);
5932 Bytes.resize(NumElements * BytesPerElement, -1);
5933 for (unsigned I = 0; I < NumElements; ++I)
5934 for (unsigned J = 0; J < BytesPerElement; ++J)
5935 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5936 return true;
5937 }
5938 return false;
5939}
5940
5941// Bytes is a VPERM-like permute vector, except that -1 is used for
5942// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5943// the result come from a contiguous sequence of bytes from one input.
5944// Set Base to the selector for the first byte if so.
5945static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5946 unsigned BytesPerElement, int &Base) {
5947 Base = -1;
5948 for (unsigned I = 0; I < BytesPerElement; ++I) {
5949 if (Bytes[Start + I] >= 0) {
5950 unsigned Elem = Bytes[Start + I];
5951 if (Base < 0) {
5952 Base = Elem - I;
5953 // Make sure the bytes would come from one input operand.
5954 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5955 return false;
5956 } else if (unsigned(Base) != Elem - I)
5957 return false;
5958 }
5959 }
5960 return true;
5961}
5962
5963// Bytes is a VPERM-like permute vector, except that -1 is used for
5964// undefined bytes. Return true if it can be performed using VSLDB.
5965// When returning true, set StartIndex to the shift amount and OpNo0
5966// and OpNo1 to the VPERM operands that should be used as the first
5967// and second shift operand respectively.
5969 unsigned &StartIndex, unsigned &OpNo0,
5970 unsigned &OpNo1) {
5971 int OpNos[] = { -1, -1 };
5972 int Shift = -1;
5973 for (unsigned I = 0; I < 16; ++I) {
5974 int Index = Bytes[I];
5975 if (Index >= 0) {
5976 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5977 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5978 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5979 if (Shift < 0)
5980 Shift = ExpectedShift;
5981 else if (Shift != ExpectedShift)
5982 return false;
5983 // Make sure that the operand mappings are consistent with previous
5984 // elements.
5985 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5986 return false;
5987 OpNos[ModelOpNo] = RealOpNo;
5988 }
5989 }
5990 StartIndex = Shift;
5991 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5992}
5993
5994// Create a node that performs P on operands Op0 and Op1, casting the
5995// operands to the appropriate type. The type of the result is determined by P.
5997 const Permute &P, SDValue Op0, SDValue Op1) {
5998 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5999 // elements of a PACK are twice as wide as the outputs.
6000 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
6001 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
6002 P.Operand);
6003 // Cast both operands to the appropriate type.
6004 MVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBytes * 8),
6005 SystemZ::VectorBytes / InBytes);
6006 Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
6007 Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
6008 SDValue Op;
6009 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
6010 SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32);
6011 Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
6012 } else if (P.Opcode == SystemZISD::PACK) {
6013 MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8),
6014 SystemZ::VectorBytes / P.Operand);
6015 Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
6016 } else {
6017 Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
6018 }
6019 return Op;
6020}
6021
6022static bool isZeroVector(SDValue N) {
6023 if (N->getOpcode() == ISD::BITCAST)
6024 N = N->getOperand(0);
6025 if (N->getOpcode() == ISD::SPLAT_VECTOR)
6026 if (auto *Op = dyn_cast<ConstantSDNode>(N->getOperand(0)))
6027 return Op->getZExtValue() == 0;
6029}
6030
6031// Return the index of the zero/undef vector, or UINT32_MAX if not found.
6032static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
6033 for (unsigned I = 0; I < Num ; I++)
6034 if (isZeroVector(Ops[I]))
6035 return I;
6036 return UINT32_MAX;
6037}
6038
6039// Bytes is a VPERM-like permute vector, except that -1 is used for
6040// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
6041// VSLDB or VPERM.
6043 SDValue *Ops,
6044 const SmallVectorImpl<int> &Bytes) {
6045 for (unsigned I = 0; I < 2; ++I)
6046 Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
6047
6048 // First see whether VSLDB can be used.
6049 unsigned StartIndex, OpNo0, OpNo1;
6050 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
6051 return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
6052 Ops[OpNo1],
6053 DAG.getTargetConstant(StartIndex, DL, MVT::i32));
6054
6055 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
6056 // eliminate a zero vector by reusing any zero index in the permute vector.
6057 unsigned ZeroVecIdx = findZeroVectorIdx(&Ops[0], 2);
6058 if (ZeroVecIdx != UINT32_MAX) {
6059 bool MaskFirst = true;
6060 int ZeroIdx = -1;
6061 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
6062 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6063 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
6064 if (OpNo == ZeroVecIdx && I == 0) {
6065 // If the first byte is zero, use mask as first operand.
6066 ZeroIdx = 0;
6067 break;
6068 }
6069 if (OpNo != ZeroVecIdx && Byte == 0) {
6070 // If mask contains a zero, use it by placing that vector first.
6071 ZeroIdx = I + SystemZ::VectorBytes;
6072 MaskFirst = false;
6073 break;
6074 }
6075 }
6076 if (ZeroIdx != -1) {
6077 SDValue IndexNodes[SystemZ::VectorBytes];
6078 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
6079 if (Bytes[I] >= 0) {
6080 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6081 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
6082 if (OpNo == ZeroVecIdx)
6083 IndexNodes[I] = DAG.getConstant(ZeroIdx, DL, MVT::i32);
6084 else {
6085 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
6086 IndexNodes[I] = DAG.getConstant(BIdx, DL, MVT::i32);
6087 }
6088 } else
6089 IndexNodes[I] = DAG.getUNDEF(MVT::i32);
6090 }
6091 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
6092 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
6093 if (MaskFirst)
6094 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Mask, Src,
6095 Mask);
6096 else
6097 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Src, Mask,
6098 Mask);
6099 }
6100 }
6101
6102 SDValue IndexNodes[SystemZ::VectorBytes];
6103 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6104 if (Bytes[I] >= 0)
6105 IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
6106 else
6107 IndexNodes[I] = DAG.getUNDEF(MVT::i32);
6108 SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
6109 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0],
6110 (!Ops[1].isUndef() ? Ops[1] : Ops[0]), Op2);
6111}
6112
6113namespace {
6114// Describes a general N-operand vector shuffle.
6115struct GeneralShuffle {
6116 GeneralShuffle(EVT vt)
6117 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(false) {}
6118 void addUndef();
6119 bool add(SDValue, unsigned);
6120 SDValue getNode(SelectionDAG &, const SDLoc &);
6121 void tryPrepareForUnpack();
6122 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
6123 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
6124
6125 // The operands of the shuffle.
6127
6128 // Index I is -1 if byte I of the result is undefined. Otherwise the
6129 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
6130 // Bytes[I] / SystemZ::VectorBytes.
6132
6133 // The type of the shuffle result.
6134 EVT VT;
6135
6136 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
6137 unsigned UnpackFromEltSize;
6138 // True if the final unpack uses the low half.
6139 bool UnpackLow;
6140};
6141} // namespace
6142
6143// Add an extra undefined element to the shuffle.
6144void GeneralShuffle::addUndef() {
6145 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6146 for (unsigned I = 0; I < BytesPerElement; ++I)
6147 Bytes.push_back(-1);
6148}
6149
6150// Add an extra element to the shuffle, taking it from element Elem of Op.
6151// A null Op indicates a vector input whose value will be calculated later;
6152// there is at most one such input per shuffle and it always has the same
6153// type as the result. Aborts and returns false if the source vector elements
6154// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
6155// LLVM they become implicitly extended, but this is rare and not optimized.
6156bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
6157 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6158
6159 // The source vector can have wider elements than the result,
6160 // either through an explicit TRUNCATE or because of type legalization.
6161 // We want the least significant part.
6162 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
6163 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
6164
6165 // Return false if the source elements are smaller than their destination
6166 // elements.
6167 if (FromBytesPerElement < BytesPerElement)
6168 return false;
6169
6170 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
6171 (FromBytesPerElement - BytesPerElement));
6172
6173 // Look through things like shuffles and bitcasts.
6174 while (Op.getNode()) {
6175 if (Op.getOpcode() == ISD::BITCAST)
6176 Op = Op.getOperand(0);
6177 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
6178 // See whether the bytes we need come from a contiguous part of one
6179 // operand.
6181 if (!getVPermMask(Op, OpBytes))
6182 break;
6183 int NewByte;
6184 if (!getShuffleInput(OpBytes, Byte, BytesPerElement, NewByte))
6185 break;
6186 if (NewByte < 0) {
6187 addUndef();
6188 return true;
6189 }
6190 Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes);
6191 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
6192 } else if (Op.isUndef()) {
6193 addUndef();
6194 return true;
6195 } else
6196 break;
6197 }
6198
6199 // Make sure that the source of the extraction is in Ops.
6200 unsigned OpNo = 0;
6201 for (; OpNo < Ops.size(); ++OpNo)
6202 if (Ops[OpNo] == Op)
6203 break;
6204 if (OpNo == Ops.size())
6205 Ops.push_back(Op);
6206
6207 // Add the element to Bytes.
6208 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
6209 for (unsigned I = 0; I < BytesPerElement; ++I)
6210 Bytes.push_back(Base + I);
6211
6212 return true;
6213}
6214
6215// Return SDNodes for the completed shuffle.
6216SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
6217 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
6218
6219 if (Ops.size() == 0)
6220 return DAG.getUNDEF(VT);
6221
6222 // Use a single unpack if possible as the last operation.
6223 tryPrepareForUnpack();
6224
6225 // Make sure that there are at least two shuffle operands.
6226 if (Ops.size() == 1)
6227 Ops.push_back(DAG.getUNDEF(MVT::v16i8));
6228
6229 // Create a tree of shuffles, deferring root node until after the loop.
6230 // Try to redistribute the undefined elements of non-root nodes so that
6231 // the non-root shuffles match something like a pack or merge, then adjust
6232 // the parent node's permute vector to compensate for the new order.
6233 // Among other things, this copes with vectors like <2 x i16> that were
6234 // padded with undefined elements during type legalization.
6235 //
6236 // In the best case this redistribution will lead to the whole tree
6237 // using packs and merges. It should rarely be a loss in other cases.
6238 unsigned Stride = 1;
6239 for (; Stride * 2 < Ops.size(); Stride *= 2) {
6240 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
6241 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
6242
6243 // Create a mask for just these two operands.
6245 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6246 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
6247 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
6248 if (OpNo == I)
6249 NewBytes[J] = Byte;
6250 else if (OpNo == I + Stride)
6251 NewBytes[J] = SystemZ::VectorBytes + Byte;
6252 else
6253 NewBytes[J] = -1;
6254 }
6255 // See if it would be better to reorganize NewMask to avoid using VPERM.
6257 if (const Permute *P = matchDoublePermute(NewBytes, NewBytesMap)) {
6258 Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
6259 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
6260 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6261 if (NewBytes[J] >= 0) {
6262 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
6263 "Invalid double permute");
6264 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
6265 } else
6266 assert(NewBytesMap[J] < 0 && "Invalid double permute");
6267 }
6268 } else {
6269 // Just use NewBytes on the operands.
6270 Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
6271 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
6272 if (NewBytes[J] >= 0)
6273 Bytes[J] = I * SystemZ::VectorBytes + J;
6274 }
6275 }
6276 }
6277
6278 // Now we just have 2 inputs. Put the second operand in Ops[1].
6279 if (Stride > 1) {
6280 Ops[1] = Ops[Stride];
6281 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6282 if (Bytes[I] >= int(SystemZ::VectorBytes))
6283 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
6284 }
6285
6286 // Look for an instruction that can do the permute without resorting
6287 // to VPERM.
6288 unsigned OpNo0, OpNo1;
6289 SDValue Op;
6290 if (unpackWasPrepared() && Ops[1].isUndef())
6291 Op = Ops[0];
6292 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
6293 Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
6294 else
6295 Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
6296
6297 Op = insertUnpackIfPrepared(DAG, DL, Op);
6298
6299 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
6300}
6301
6302#ifndef NDEBUG
6303static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
6304 dbgs() << Msg.c_str() << " { ";
6305 for (unsigned I = 0; I < Bytes.size(); I++)
6306 dbgs() << Bytes[I] << " ";
6307 dbgs() << "}\n";
6308}
6309#endif
6310
6311// If the Bytes vector matches an unpack operation, prepare to do the unpack
6312// after all else by removing the zero vector and the effect of the unpack on
6313// Bytes.
6314void GeneralShuffle::tryPrepareForUnpack() {
6315 uint32_t ZeroVecOpNo = findZeroVectorIdx(&Ops[0], Ops.size());
6316 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
6317 return;
6318
6319 // Only do this if removing the zero vector reduces the depth, otherwise
6320 // the critical path will increase with the final unpack.
6321 if (Ops.size() > 2 &&
6322 Log2_32_Ceil(Ops.size()) == Log2_32_Ceil(Ops.size() - 1))
6323 return;
6324
6325 // Find an unpack that would allow removing the zero vector from Ops.
6326 UnpackFromEltSize = 1;
6327 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6328 bool MatchUnpack = true;
6330 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
6331 unsigned ToEltSize = UnpackFromEltSize * 2;
6332 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6333 if (!IsZextByte)
6334 SrcBytes.push_back(Bytes[Elt]);
6335 if (Bytes[Elt] != -1) {
6336 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
6337 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6338 MatchUnpack = false;
6339 break;
6340 }
6341 }
6342 }
6343 if (MatchUnpack) {
6344 if (Ops.size() == 2) {
6345 // Don't use unpack if a single source operand needs rearrangement.
6346 bool CanUseUnpackLow = true, CanUseUnpackHigh = true;
6347 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++) {
6348 if (SrcBytes[i] == -1)
6349 continue;
6350 if (SrcBytes[i] % 16 != int(i))
6351 CanUseUnpackHigh = false;
6352 if (SrcBytes[i] % 16 != int(i + SystemZ::VectorBytes / 2))
6353 CanUseUnpackLow = false;
6354 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6355 UnpackFromEltSize = UINT_MAX;
6356 return;
6357 }
6358 }
6359 if (!CanUseUnpackHigh)
6360 UnpackLow = true;
6361 }
6362 break;
6363 }
6364 }
6365 if (UnpackFromEltSize > 4)
6366 return;
6367
6368 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
6369 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
6370 << ".\n";
6371 dumpBytes(Bytes, "Original Bytes vector:"););
6372
6373 // Apply the unpack in reverse to the Bytes array.
6374 unsigned B = 0;
6375 if (UnpackLow) {
6376 while (B < SystemZ::VectorBytes / 2)
6377 Bytes[B++] = -1;
6378 }
6379 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
6380 Elt += UnpackFromEltSize;
6381 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
6382 Bytes[B] = Bytes[Elt];
6383 }
6384 if (!UnpackLow) {
6385 while (B < SystemZ::VectorBytes)
6386 Bytes[B++] = -1;
6387 }
6388
6389 // Remove the zero vector from Ops
6390 Ops.erase(&Ops[ZeroVecOpNo]);
6391 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6392 if (Bytes[I] >= 0) {
6393 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6394 if (OpNo > ZeroVecOpNo)
6395 Bytes[I] -= SystemZ::VectorBytes;
6396 }
6397
6398 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
6399 dbgs() << "\n";);
6400}
6401
6402SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
6403 const SDLoc &DL,
6404 SDValue Op) {
6405 if (!unpackWasPrepared())
6406 return Op;
6407 unsigned InBits = UnpackFromEltSize * 8;
6408 EVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBits),
6409 SystemZ::VectorBits / InBits);
6410 SDValue PackedOp = DAG.getNode(ISD::BITCAST, DL, InVT, Op);
6411 unsigned OutBits = InBits * 2;
6412 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(OutBits),
6413 SystemZ::VectorBits / OutBits);
6414 return DAG.getNode(UnpackLow ? SystemZISD::UNPACKL_LOW
6415 : SystemZISD::UNPACKL_HIGH,
6416 DL, OutVT, PackedOp);
6417}
6418
6419// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
6421 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
6422 if (!Op.getOperand(I).isUndef())
6423 return false;
6424 return true;
6425}
6426
6427// Return a vector of type VT that contains Value in the first element.
6428// The other elements don't matter.
6430 SDValue Value) {
6431 // If we have a constant, replicate it to all elements and let the
6432 // BUILD_VECTOR lowering take care of it.
6433 if (Value.getOpcode() == ISD::Constant ||
6434 Value.getOpcode() == ISD::ConstantFP) {
6436 return DAG.getBuildVector(VT, DL, Ops);
6437 }
6438 if (Value.isUndef())
6439 return DAG.getUNDEF(VT);
6440 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
6441}
6442
6443// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
6444// element 1. Used for cases in which replication is cheap.
6446 SDValue Op0, SDValue Op1) {
6447 if (Op0.isUndef()) {
6448 if (Op1.isUndef())
6449 return DAG.getUNDEF(VT);
6450 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
6451 }
6452 if (Op1.isUndef())
6453 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
6454 return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
6455 buildScalarToVector(DAG, DL, VT, Op0),
6456 buildScalarToVector(DAG, DL, VT, Op1));
6457}
6458
6459// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
6460// vector for them.
6462 SDValue Op1) {
6463 if (Op0.isUndef() && Op1.isUndef())
6464 return DAG.getUNDEF(MVT::v2i64);
6465 // If one of the two inputs is undefined then replicate the other one,
6466 // in order to avoid using another register unnecessarily.
6467 if (Op0.isUndef())
6468 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
6469 else if (Op1.isUndef())
6470 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
6471 else {
6472 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
6473 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
6474 }
6475 return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
6476}
6477
6478// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
6479// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
6480// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
6481// would benefit from this representation and return it if so.
6483 BuildVectorSDNode *BVN) {
6484 EVT VT = BVN->getValueType(0);
6485 unsigned NumElements = VT.getVectorNumElements();
6486
6487 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
6488 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
6489 // need a BUILD_VECTOR, add an additional placeholder operand for that
6490 // BUILD_VECTOR and store its operands in ResidueOps.
6491 GeneralShuffle GS(VT);
6493 bool FoundOne = false;
6494 for (unsigned I = 0; I < NumElements; ++I) {
6495 SDValue Op = BVN->getOperand(I);
6496 if (Op.getOpcode() == ISD::TRUNCATE)
6497 Op = Op.getOperand(0);
6498 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6499 Op.getOperand(1).getOpcode() == ISD::Constant) {
6500 unsigned Elem = Op.getConstantOperandVal(1);
6501 if (!GS.add(Op.getOperand(0), Elem))
6502 return SDValue();
6503 FoundOne = true;
6504 } else if (Op.isUndef()) {
6505 GS.addUndef();
6506 } else {
6507 if (!GS.add(SDValue(), ResidueOps.size()))
6508 return SDValue();
6509 ResidueOps.push_back(BVN->getOperand(I));
6510 }
6511 }
6512
6513 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
6514 if (!FoundOne)
6515 return SDValue();
6516
6517 // Create the BUILD_VECTOR for the remaining elements, if any.
6518 if (!ResidueOps.empty()) {
6519 while (ResidueOps.size() < NumElements)
6520 ResidueOps.push_back(DAG.getUNDEF(ResidueOps[0].getValueType()));
6521 for (auto &Op : GS.Ops) {
6522 if (!Op.getNode()) {
6523 Op = DAG.getBuildVector(VT, SDLoc(BVN), ResidueOps);
6524 break;
6525 }
6526 }
6527 }
6528 return GS.getNode(DAG, SDLoc(BVN));
6529}
6530
6531bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
6532 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Op)->isUnindexed())
6533 return true;
6534 if (auto *AL = dyn_cast<AtomicSDNode>(Op))
6535 if (AL->getOpcode() == ISD::ATOMIC_LOAD)
6536 return true;
6537 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
6538 return true;
6539 return false;
6540}
6541
6543 unsigned MergedBits, EVT VT, SDValue Op0,
6544 SDValue Op1) {
6545 MVT IntVecVT = MVT::getVectorVT(MVT::getIntegerVT(MergedBits),
6546 SystemZ::VectorBits / MergedBits);
6547 assert(VT.getSizeInBits() == 128 && IntVecVT.getSizeInBits() == 128 &&
6548 "Handling full vectors only.");
6549 Op0 = DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0);
6550 Op1 = DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op1);
6551 SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH, DL, IntVecVT, Op0, Op1);
6552 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
6553}
6554
6556 EVT VT, SmallVectorImpl<SDValue> &Elems,
6557 unsigned Pos) {
6558 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[Pos + 0], Elems[Pos + 1]);
6559 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[Pos + 2], Elems[Pos + 3]);
6560 // Avoid unnecessary undefs by reusing the other operand.
6561 if (Op01.isUndef()) {
6562 if (Op23.isUndef())
6563 return Op01;
6564 Op01 = Op23;
6565 } else if (Op23.isUndef())
6566 Op23 = Op01;
6567 // Merging identical replications is a no-op.
6568 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6569 return Op01;
6570 unsigned MergedBits = VT.getSimpleVT().getScalarSizeInBits() * 2;
6571 return mergeHighParts(DAG, DL, MergedBits, VT, Op01, Op23);
6572}
6573
6574// Combine GPR scalar values Elems into a vector of type VT.
6575SDValue
6576SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6577 SmallVectorImpl<SDValue> &Elems) const {
6578 // See whether there is a single replicated value.
6580 unsigned int NumElements = Elems.size();
6581 unsigned int Count = 0;
6582 for (auto Elem : Elems) {
6583 if (!Elem.isUndef()) {
6584 if (!Single.getNode())
6585 Single = Elem;
6586 else if (Elem != Single) {
6587 Single = SDValue();
6588 break;
6589 }
6590 Count += 1;
6591 }
6592 }
6593 // There are three cases here:
6594 //
6595 // - if the only defined element is a loaded one, the best sequence
6596 // is a replicating load.
6597 //
6598 // - otherwise, if the only defined element is an i64 value, we will
6599 // end up with the same VLVGP sequence regardless of whether we short-cut
6600 // for replication or fall through to the later code.
6601 //
6602 // - otherwise, if the only defined element is an i32 or smaller value,
6603 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
6604 // This is only a win if the single defined element is used more than once.
6605 // In other cases we're better off using a single VLVGx.
6606 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Single)))
6607 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
6608
6609 // If all elements are loads, use VLREP/VLEs (below).
6610 bool AllLoads = true;
6611 for (auto Elem : Elems)
6612 if (!isVectorElementLoad(Elem)) {
6613 AllLoads = false;
6614 break;
6615 }
6616
6617 // The best way of building a v2i64 from two i64s is to use VLVGP.
6618 if (VT == MVT::v2i64 && !AllLoads)
6619 return joinDwords(DAG, DL, Elems[0], Elems[1]);
6620
6621 // Use a 64-bit merge high to combine two doubles.
6622 if (VT == MVT::v2f64 && !AllLoads)
6623 return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
6624
6625 // Build v4f32 values directly from the FPRs:
6626 //
6627 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
6628 // V V VMRHF
6629 // <ABxx> <CDxx>
6630 // V VMRHG
6631 // <ABCD>
6632 if (VT == MVT::v4f32 && !AllLoads)
6633 return buildFPVecFromScalars4(DAG, DL, VT, Elems, 0);
6634
6635 // Same for v8f16.
6636 if (VT == MVT::v8f16 && !AllLoads) {
6637 SDValue Op0123 = buildFPVecFromScalars4(DAG, DL, VT, Elems, 0);
6638 SDValue Op4567 = buildFPVecFromScalars4(DAG, DL, VT, Elems, 4);
6639 // Avoid unnecessary undefs by reusing the other operand.
6640 if (Op0123.isUndef())
6641 Op0123 = Op4567;
6642 else if (Op4567.isUndef())
6643 Op4567 = Op0123;
6644 // Merging identical replications is a no-op.
6645 if (Op0123.getOpcode() == SystemZISD::REPLICATE && Op0123 == Op4567)
6646 return Op0123;
6647 return mergeHighParts(DAG, DL, 64, VT, Op0123, Op4567);
6648 }
6649
6650 // Collect the constant terms.
6653
6654 unsigned NumConstants = 0;
6655 for (unsigned I = 0; I < NumElements; ++I) {
6656 SDValue Elem = Elems[I];
6657 if (Elem.getOpcode() == ISD::Constant ||
6658 Elem.getOpcode() == ISD::ConstantFP) {
6659 NumConstants += 1;
6660 Constants[I] = Elem;
6661 Done[I] = true;
6662 }
6663 }
6664 // If there was at least one constant, fill in the other elements of
6665 // Constants with undefs to get a full vector constant and use that
6666 // as the starting point.
6668 SDValue ReplicatedVal;
6669 if (NumConstants > 0) {
6670 for (unsigned I = 0; I < NumElements; ++I)
6671 if (!Constants[I].getNode())
6672 Constants[I] = DAG.getUNDEF(Elems[I].getValueType());
6673 Result = DAG.getBuildVector(VT, DL, Constants);
6674 } else {
6675 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
6676 // avoid a false dependency on any previous contents of the vector
6677 // register.
6678
6679 // Use a VLREP if at least one element is a load. Make sure to replicate
6680 // the load with the most elements having its value.
6681 std::map<const SDNode*, unsigned> UseCounts;
6682 SDNode *LoadMaxUses = nullptr;
6683 for (unsigned I = 0; I < NumElements; ++I)
6684 if (isVectorElementLoad(Elems[I])) {
6685 SDNode *Ld = Elems[I].getNode();
6686 unsigned Count = ++UseCounts[Ld];
6687 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
6688 LoadMaxUses = Ld;
6689 }
6690 if (LoadMaxUses != nullptr) {
6691 ReplicatedVal = SDValue(LoadMaxUses, 0);
6692 Result = DAG.getNode(SystemZISD::REPLICATE, DL, VT, ReplicatedVal);
6693 } else {
6694 // Try to use VLVGP.
6695 unsigned I1 = NumElements / 2 - 1;
6696 unsigned I2 = NumElements - 1;
6697 bool Def1 = !Elems[I1].isUndef();
6698 bool Def2 = !Elems[I2].isUndef();
6699 if (Def1 || Def2) {
6700 SDValue Elem1 = Elems[Def1 ? I1 : I2];
6701 SDValue Elem2 = Elems[Def2 ? I2 : I1];
6702 Result = DAG.getNode(ISD::BITCAST, DL, VT,
6703 joinDwords(DAG, DL, Elem1, Elem2));
6704 Done[I1] = true;
6705 Done[I2] = true;
6706 } else
6707 Result = DAG.getUNDEF(VT);
6708 }
6709 }
6710
6711 // Use VLVGx to insert the other elements.
6712 for (unsigned I = 0; I < NumElements; ++I)
6713 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
6714 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I],
6715 DAG.getConstant(I, DL, MVT::i32));
6716 return Result;
6717}
6718
6719SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
6720 SelectionDAG &DAG) const {
6721 auto *BVN = cast<BuildVectorSDNode>(Op.getNode());
6722 SDLoc DL(Op);
6723 EVT VT = Op.getValueType();
6724
6725 if (BVN->isConstant()) {
6726 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6727 return Op;
6728
6729 // Fall back to loading it from memory.
6730 return SDValue();
6731 }
6732
6733 // See if we should use shuffles to construct the vector from other vectors.
6734 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6735 return Res;
6736
6737 // Detect SCALAR_TO_VECTOR conversions.
6739 return buildScalarToVector(DAG, DL, VT, Op.getOperand(0));
6740
6741 // Otherwise use buildVector to build the vector up from GPRs.
6742 unsigned NumElements = Op.getNumOperands();
6744 for (unsigned I = 0; I < NumElements; ++I)
6745 Ops[I] = Op.getOperand(I);
6746 return buildVector(DAG, DL, VT, Ops);
6747}
6748
6749SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6750 SelectionDAG &DAG) const {
6751 auto *VSN = cast<ShuffleVectorSDNode>(Op.getNode());
6752 SDLoc DL(Op);
6753 EVT VT = Op.getValueType();
6754 unsigned NumElements = VT.getVectorNumElements();
6755
6756 if (VSN->isSplat()) {
6757 SDValue Op0 = Op.getOperand(0);
6758 unsigned Index = VSN->getSplatIndex();
6759 assert(Index < VT.getVectorNumElements() &&
6760 "Splat index should be defined and in first operand");
6761 // See whether the value we're splatting is directly available as a scalar.
6762 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6764 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
6765 // Otherwise keep it as a vector-to-vector operation.
6766 return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
6767 DAG.getTargetConstant(Index, DL, MVT::i32));
6768 }
6769
6770 GeneralShuffle GS(VT);
6771 for (unsigned I = 0; I < NumElements; ++I) {
6772 int Elt = VSN->getMaskElt(I);
6773 if (Elt < 0)
6774 GS.addUndef();
6775 else if (!GS.add(Op.getOperand(unsigned(Elt) / NumElements),
6776 unsigned(Elt) % NumElements))
6777 return SDValue();
6778 }
6779 return GS.getNode(DAG, SDLoc(VSN));
6780}
6781
6782SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
6783 SelectionDAG &DAG) const {
6784 SDLoc DL(Op);
6785 // Just insert the scalar into element 0 of an undefined vector.
6786 return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL,
6787 Op.getValueType(), DAG.getUNDEF(Op.getValueType()),
6788 Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32));
6789}
6790
6791// Shift the lower 2 bytes of Op to the left in order to insert into the
6792// upper 2 bytes of the FP register.
6794 assert(Op.getSimpleValueType() == MVT::i64 &&
6795 "Expexted to convert i64 to f16.");
6796 SDLoc DL(Op);
6797 SDValue Shft = DAG.getNode(ISD::SHL, DL, MVT::i64, Op,
6798 DAG.getConstant(48, DL, MVT::i64));
6799 SDValue BCast = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shft);
6800 SDValue F16Val =
6801 DAG.getTargetExtractSubreg(SystemZ::subreg_h16, DL, MVT::f16, BCast);
6802 return F16Val;
6803}
6804
6805// Extract Op into GPR and shift the 2 f16 bytes to the right.
6807 assert(Op.getSimpleValueType() == MVT::f16 &&
6808 "Expected to convert f16 to i64.");
6809 SDNode *U32 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
6810 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h16, DL, MVT::f64,
6811 SDValue(U32, 0), Op);
6812 SDValue BCast = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
6813 SDValue Shft = DAG.getNode(ISD::SRL, DL, MVT::i64, BCast,
6814 DAG.getConstant(48, DL, MVT::i32));
6815 return Shft;
6816}
6817
6819 auto *CIdx = dyn_cast<ConstantSDNode>(Idx);
6820 return CIdx && CIdx->getZExtValue() < VecVT.getVectorNumElements();
6821}
6822
6823SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
6824 SelectionDAG &DAG) const {
6825 // Handle insertions of floating-point values.
6826 SDLoc DL(Op);
6827 SDValue Op0 = Op.getOperand(0);
6828 SDValue Op1 = Op.getOperand(1);
6829 SDValue Op2 = Op.getOperand(2);
6830 EVT VT = Op.getValueType();
6831
6832 // A constant index selects a GR128 subregister.
6833 if (!Subtarget.hasVector())
6834 return isInBoundsConstantVectorIndex(VT, Op2) ? Op : SDValue();
6835
6836 // Insertions into constant indices of a v2f64 can be done using VPDI.
6837 // However, if the inserted value is a bitcast or a constant then it's
6838 // better to use GPRs, as below.
6839 if (VT == MVT::v2f64 && Op1.getOpcode() != ISD::BITCAST &&
6840 Op1.getOpcode() != ISD::ConstantFP &&
6842 return Op;
6843
6844 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
6845 MVT IntVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
6846 MVT IntVecVT = MVT::getVectorVT(IntVT, VT.getVectorNumElements());
6847 SDValue IntOp1 =
6848 VT == MVT::v8f16
6849 ? DAG.getZExtOrTrunc(convertFromF16(Op1, DL, DAG), DL, MVT::i32)
6850 : DAG.getNode(ISD::BITCAST, DL, IntVT, Op1);
6851 SDValue Res =
6852 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
6853 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), IntOp1, Op2);
6854 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
6855}
6856
6857SDValue
6858SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
6859 SelectionDAG &DAG) const {
6860 // Handle extractions of floating-point values.
6861 SDLoc DL(Op);
6862 SDValue Op0 = Op.getOperand(0);
6863 SDValue Op1 = Op.getOperand(1);
6864 EVT VT = Op.getValueType();
6865 EVT VecVT = Op0.getValueType();
6866
6867 // Extractions of constant indices can be done directly.
6868 if (isInBoundsConstantVectorIndex(VecVT, Op1))
6869 return Op;
6870
6871 if (!Subtarget.hasVector())
6872 return SDValue();
6873
6874 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
6875 MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
6876 MVT IntVecVT = MVT::getVectorVT(IntVT, VecVT.getVectorNumElements());
6877 MVT ExtrVT = IntVT == MVT::i16 ? MVT::i32 : IntVT;
6878 SDValue Extr = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrVT,
6879 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1);
6880 if (VT == MVT::f16)
6881 return convertToF16(DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Extr), DAG);
6882 return DAG.getNode(ISD::BITCAST, DL, VT, Extr);
6883}
6884
6885SDValue SystemZTargetLowering::
6886lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6887 SDValue PackedOp = Op.getOperand(0);
6888 EVT OutVT = Op.getValueType();
6889 EVT InVT = PackedOp.getValueType();
6890 unsigned ToBits = OutVT.getScalarSizeInBits();
6891 unsigned FromBits = InVT.getScalarSizeInBits();
6892 unsigned StartOffset = 0;
6893
6894 // If the input is a VECTOR_SHUFFLE, there are a number of important
6895 // cases where we can directly implement the sign-extension of the
6896 // original input lanes of the shuffle.
6897 if (PackedOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
6898 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(PackedOp.getNode());
6899 ArrayRef<int> ShuffleMask = SVN->getMask();
6900 int OutNumElts = OutVT.getVectorNumElements();
6901
6902 // Recognize the special case where the sign-extension can be done
6903 // by the VSEG instruction. Handled via the default expander.
6904 if (ToBits == 64 && OutNumElts == 2) {
6905 int NumElem = ToBits / FromBits;
6906 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6907 return SDValue();
6908 }
6909
6910 // Recognize the special case where we can fold the shuffle by
6911 // replacing some of the UNPACK_HIGH with UNPACK_LOW.
6912 int StartOffsetCandidate = -1;
6913 for (int Elt = 0; Elt < OutNumElts; Elt++) {
6914 if (ShuffleMask[Elt] == -1)
6915 continue;
6916 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6917 if (StartOffsetCandidate == -1)
6918 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6919 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6920 continue;
6921 }
6922 StartOffsetCandidate = -1;
6923 break;
6924 }
6925 if (StartOffsetCandidate != -1) {
6926 // The mask indexes the concatenation of both shuffle operands: an offset
6927 // past the first operand's lanes selects lanes of the second.
6928 unsigned InNumElts = InVT.getVectorNumElements();
6929 unsigned Offset = StartOffsetCandidate;
6930 unsigned OpNo = Offset >= InNumElts ? 1 : 0;
6931 StartOffset = Offset - OpNo * InNumElts;
6932 PackedOp = PackedOp.getOperand(OpNo);
6933 }
6934 }
6935
6936 do {
6937 FromBits *= 2;
6938 unsigned OutNumElts = SystemZ::VectorBits / FromBits;
6939 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(FromBits), OutNumElts);
6940 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6941 if (StartOffset >= OutNumElts) {
6942 Opcode = SystemZISD::UNPACK_LOW;
6943 StartOffset -= OutNumElts;
6944 }
6945 PackedOp = DAG.getNode(Opcode, SDLoc(PackedOp), OutVT, PackedOp);
6946 } while (FromBits != ToBits);
6947 return PackedOp;
6948}
6949
6950// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
6951SDValue SystemZTargetLowering::
6952lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6953 SDValue PackedOp = Op.getOperand(0);
6954 SDLoc DL(Op);
6955 EVT OutVT = Op.getValueType();
6956 EVT InVT = PackedOp.getValueType();
6957 unsigned InNumElts = InVT.getVectorNumElements();
6958 unsigned OutNumElts = OutVT.getVectorNumElements();
6959 unsigned NumInPerOut = InNumElts / OutNumElts;
6960
6961 SDValue ZeroVec =
6962 DAG.getSplatVector(InVT, DL, DAG.getConstant(0, DL, InVT.getScalarType()));
6963
6964 SmallVector<int, 16> Mask(InNumElts);
6965 unsigned ZeroVecElt = InNumElts;
6966 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6967 unsigned MaskElt = PackedElt * NumInPerOut;
6968 unsigned End = MaskElt + NumInPerOut - 1;
6969 for (; MaskElt < End; MaskElt++)
6970 Mask[MaskElt] = ZeroVecElt++;
6971 Mask[MaskElt] = PackedElt;
6972 }
6973 SDValue Shuf = DAG.getVectorShuffle(InVT, DL, PackedOp, ZeroVec, Mask);
6974 return DAG.getNode(ISD::BITCAST, DL, OutVT, Shuf);
6975}
6976
6977SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6978 unsigned ByScalar) const {
6979 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6980 SDValue Op0 = Op.getOperand(0);
6981 SDValue Op1 = Op.getOperand(1);
6982 SDLoc DL(Op);
6983 EVT VT = Op.getValueType();
6984 unsigned ElemBitSize = VT.getScalarSizeInBits();
6985
6986 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6987 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op1)) {
6988 APInt SplatBits, SplatUndef;
6989 unsigned SplatBitSize;
6990 bool HasAnyUndefs;
6991 // Check for constant splats. Use ElemBitSize as the minimum element
6992 // width and reject splats that need wider elements.
6993 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6994 ElemBitSize, true) &&
6995 SplatBitSize == ElemBitSize) {
6996 SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff,
6997 DL, MVT::i32);
6998 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
6999 }
7000 // Check for variable splats.
7001 BitVector UndefElements;
7002 SDValue Splat = BVN->getSplatValue(&UndefElements);
7003 if (Splat) {
7004 // Since i32 is the smallest legal type, we either need a no-op
7005 // or a truncation.
7006 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
7007 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
7008 }
7009 }
7010
7011 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
7012 // and the shift amount is directly available in a GPR.
7013 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Op1)) {
7014 if (VSN->isSplat()) {
7015 SDValue VSNOp0 = VSN->getOperand(0);
7016 unsigned Index = VSN->getSplatIndex();
7017 assert(Index < VT.getVectorNumElements() &&
7018 "Splat index should be defined and in first operand");
7019 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
7020 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
7021 // Since i32 is the smallest legal type, we either need a no-op
7022 // or a truncation.
7023 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
7024 VSNOp0.getOperand(Index));
7025 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
7026 }
7027 }
7028 }
7029
7030 // Otherwise just treat the current form as legal.
7031 return Op;
7032}
7033
7034SDValue SystemZTargetLowering::lowerFSHL(SDValue Op, SelectionDAG &DAG) const {
7035 SDLoc DL(Op);
7036
7037 // i128 FSHL with a constant amount that is a multiple of 8 can be
7038 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
7039 // facility, FSHL with a constant amount less than 8 can be implemented
7040 // via SHL_DOUBLE_BIT, and FSHL with other constant amounts by a
7041 // combination of the two.
7042 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Op.getOperand(2))) {
7043 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
7044 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
7045 SDValue Op0 = DAG.getBitcast(MVT::v16i8, Op.getOperand(0));
7046 SDValue Op1 = DAG.getBitcast(MVT::v16i8, Op.getOperand(1));
7047 if (ShiftAmt > 120) {
7048 // For N in 121..128, fshl N == fshr (128 - N), and for 1 <= N < 8
7049 // SHR_DOUBLE_BIT emits fewer instructions.
7050 SDValue Val =
7051 DAG.getNode(SystemZISD::SHR_DOUBLE_BIT, DL, MVT::v16i8, Op0, Op1,
7052 DAG.getTargetConstant(128 - ShiftAmt, DL, MVT::i32));
7053 return DAG.getBitcast(MVT::i128, Val);
7054 }
7055 SmallVector<int, 16> Mask(16);
7056 for (unsigned Elt = 0; Elt < 16; Elt++)
7057 Mask[Elt] = (ShiftAmt >> 3) + Elt;
7058 SDValue Shuf1 = DAG.getVectorShuffle(MVT::v16i8, DL, Op0, Op1, Mask);
7059 if ((ShiftAmt & 7) == 0)
7060 return DAG.getBitcast(MVT::i128, Shuf1);
7061 SDValue Shuf2 = DAG.getVectorShuffle(MVT::v16i8, DL, Op1, Op1, Mask);
7062 SDValue Val =
7063 DAG.getNode(SystemZISD::SHL_DOUBLE_BIT, DL, MVT::v16i8, Shuf1, Shuf2,
7064 DAG.getTargetConstant(ShiftAmt & 7, DL, MVT::i32));
7065 return DAG.getBitcast(MVT::i128, Val);
7066 }
7067 }
7068
7069 return SDValue();
7070}
7071
7072SDValue SystemZTargetLowering::lowerFSHR(SDValue Op, SelectionDAG &DAG) const {
7073 SDLoc DL(Op);
7074
7075 // i128 FSHR with a constant amount that is a multiple of 8 can be
7076 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
7077 // facility, FSHR with a constant amount less than 8 can be implemented
7078 // via SHR_DOUBLE_BIT, and FSHR with other constant amounts by a
7079 // combination of the two.
7080 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Op.getOperand(2))) {
7081 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
7082 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
7083 SDValue Op0 = DAG.getBitcast(MVT::v16i8, Op.getOperand(0));
7084 SDValue Op1 = DAG.getBitcast(MVT::v16i8, Op.getOperand(1));
7085 if (ShiftAmt > 120) {
7086 // For N in 121..128, fshr N == fshl (128 - N), and for 1 <= N < 8
7087 // SHL_DOUBLE_BIT emits fewer instructions.
7088 SDValue Val =
7089 DAG.getNode(SystemZISD::SHL_DOUBLE_BIT, DL, MVT::v16i8, Op0, Op1,
7090 DAG.getTargetConstant(128 - ShiftAmt, DL, MVT::i32));
7091 return DAG.getBitcast(MVT::i128, Val);
7092 }
7093 SmallVector<int, 16> Mask(16);
7094 for (unsigned Elt = 0; Elt < 16; Elt++)
7095 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
7096 SDValue Shuf1 = DAG.getVectorShuffle(MVT::v16i8, DL, Op0, Op1, Mask);
7097 if ((ShiftAmt & 7) == 0)
7098 return DAG.getBitcast(MVT::i128, Shuf1);
7099 SDValue Shuf2 = DAG.getVectorShuffle(MVT::v16i8, DL, Op0, Op0, Mask);
7100 SDValue Val =
7101 DAG.getNode(SystemZISD::SHR_DOUBLE_BIT, DL, MVT::v16i8, Shuf2, Shuf1,
7102 DAG.getTargetConstant(ShiftAmt & 7, DL, MVT::i32));
7103 return DAG.getBitcast(MVT::i128, Val);
7104 }
7105 }
7106
7107 return SDValue();
7108}
7109
7111 SDLoc DL(Op);
7112 SDValue Src = Op.getOperand(0);
7113 MVT DstVT = Op.getSimpleValueType();
7114
7116 unsigned SrcAS = N->getSrcAddressSpace();
7117
7118 assert(SrcAS != N->getDestAddressSpace() &&
7119 "addrspacecast must be between different address spaces");
7120
7121 // addrspacecast [0 <- 1] : Assinging a ptr32 value to a 64-bit pointer.
7122 // addrspacecast [1 <- 0] : Assigining a 64-bit pointer to a ptr32 value.
7123 if (SrcAS == SYSTEMZAS::PTR32 && DstVT == MVT::i64) {
7124 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Src,
7125 DAG.getConstant(0x7fffffff, DL, MVT::i32));
7126 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, DstVT, Op);
7127 } else if (DstVT == MVT::i32) {
7128 Op = DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src);
7129 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Op,
7130 DAG.getConstant(0x7fffffff, DL, MVT::i32));
7131 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, DstVT, Op);
7132 } else {
7133 report_fatal_error("Bad address space in addrspacecast");
7134 }
7135 return Op;
7136}
7137
7138SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
7139 SelectionDAG &DAG) const {
7140 SDValue In = Op.getOperand(Op->isStrictFPOpcode() ? 1 : 0);
7141 if (In.getSimpleValueType() != MVT::f16)
7142 return Op; // Legal
7143 return SDValue(); // Let legalizer emit the libcall.
7144}
7145
7147 MVT VT, SDValue Arg, SDLoc DL,
7148 SDValue Chain, bool IsStrict) const {
7149 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
7150 MakeLibCallOptions CallOptions;
7151 SDValue Result;
7152 std::tie(Result, Chain) =
7153 makeLibCall(DAG, LC, VT, Arg, CallOptions, DL, Chain);
7154 return IsStrict ? DAG.getMergeValues({Result, Chain}, DL) : Result;
7155}
7156
7157SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
7158 SelectionDAG &DAG) const {
7159 bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
7160 Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
7161 bool IsStrict = Op->isStrictFPOpcode();
7162 SDLoc DL(Op);
7163 MVT VT = Op.getSimpleValueType();
7164 SDValue InOp = Op.getOperand(IsStrict ? 1 : 0);
7165 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
7166 EVT InVT = InOp.getValueType();
7167
7168 // FP to unsigned is not directly supported on z10. Promoting an i32
7169 // result to (signed) i64 doesn't generate an inexact condition (fp
7170 // exception) for values that are outside the i32 range but in the i64
7171 // range, so use the default expansion.
7172 if (!Subtarget.hasFPExtension() && !IsSigned)
7173 // Expand i32/i64. F16 values will be recognized to fit and extended.
7174 return SDValue();
7175
7176 // Conversion from f16 is done via f32.
7177 if (InOp.getSimpleValueType() == MVT::f16) {
7179 LowerOperationWrapper(Op.getNode(), Results, DAG);
7180 return DAG.getMergeValues(Results, DL);
7181 }
7182
7183 if (VT == MVT::i128) {
7184 RTLIB::Libcall LC =
7185 IsSigned ? RTLIB::getFPTOSINT(InVT, VT) : RTLIB::getFPTOUINT(InVT, VT);
7186 return useLibCall(DAG, LC, VT, InOp, DL, Chain, IsStrict);
7187 }
7188
7189 return Op; // Legal
7190}
7191
7192SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
7193 SelectionDAG &DAG) const {
7194 bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
7195 Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
7196 bool IsStrict = Op->isStrictFPOpcode();
7197 SDLoc DL(Op);
7198 MVT VT = Op.getSimpleValueType();
7199 SDValue InOp = Op.getOperand(IsStrict ? 1 : 0);
7200 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
7201 EVT InVT = InOp.getValueType();
7202
7203 // Conversion to f16 is done via f32.
7204 if (VT == MVT::f16) {
7206 LowerOperationWrapper(Op.getNode(), Results, DAG);
7207 return DAG.getMergeValues(Results, DL);
7208 }
7209
7210 // Unsigned to fp is not directly supported on z10.
7211 if (!Subtarget.hasFPExtension() && !IsSigned)
7212 return SDValue(); // Expand i64.
7213
7214 if (InVT == MVT::i128) {
7215 RTLIB::Libcall LC =
7216 IsSigned ? RTLIB::getSINTTOFP(InVT, VT) : RTLIB::getUINTTOFP(InVT, VT);
7217 return useLibCall(DAG, LC, VT, InOp, DL, Chain, IsStrict);
7218 }
7219
7220 return Op; // Legal
7221}
7222
7223// Lower an f16 LOAD in case of no vector support.
7224SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
7225 SelectionDAG &DAG) const {
7226 EVT RegVT = Op.getValueType();
7227 assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
7228 (void)RegVT;
7229
7230 // Load as integer.
7231 SDLoc DL(Op);
7232 SDValue NewLd;
7233 if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Op.getNode())) {
7234 assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
7235 NewLd = DAG.getAtomicLoad(ISD::EXTLOAD, DL, MVT::i16, MVT::i64,
7236 AtomicLd->getChain(), AtomicLd->getBasePtr(),
7237 AtomicLd->getMemOperand());
7238 } else {
7239 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
7240 assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
7241 NewLd = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i64, Ld->getChain(),
7242 Ld->getBasePtr(), Ld->getPointerInfo(), MVT::i16,
7243 Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
7244 }
7245 SDValue F16Val = convertToF16(NewLd, DAG);
7246 return DAG.getMergeValues({F16Val, NewLd.getValue(1)}, DL);
7247}
7248
7249// Lower an f16 STORE in case of no vector support.
7250SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
7251 SelectionDAG &DAG) const {
7252 SDLoc DL(Op);
7253 SDValue Shft = convertFromF16(Op->getOperand(1), DL, DAG);
7254
7255 if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Op.getNode()))
7256 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, MVT::i16, AtomicSt->getChain(),
7257 Shft, AtomicSt->getBasePtr(),
7258 AtomicSt->getMemOperand());
7259
7260 StoreSDNode *St = cast<StoreSDNode>(Op.getNode());
7261 return DAG.getTruncStore(St->getChain(), DL, Shft, St->getBasePtr(), MVT::i16,
7262 St->getMemOperand());
7263}
7264
7265SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
7266 SelectionDAG &DAG) const {
7267 SDLoc DL(Op);
7268 MVT ResultVT = Op.getSimpleValueType();
7269 SDValue Arg = Op.getOperand(0);
7270 unsigned Check = Op.getConstantOperandVal(1);
7271
7272 unsigned TDCMask = 0;
7273 if (Check & fcSNan)
7275 if (Check & fcQNan)
7277 if (Check & fcPosInf)
7279 if (Check & fcNegInf)
7281 if (Check & fcPosNormal)
7283 if (Check & fcNegNormal)
7285 if (Check & fcPosSubnormal)
7287 if (Check & fcNegSubnormal)
7289 if (Check & fcPosZero)
7290 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
7291 if (Check & fcNegZero)
7292 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
7293 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, MVT::i64);
7294
7295 SDValue Intr = DAG.getNode(SystemZISD::TDC, DL, ResultVT, Arg, TDCMaskV);
7296 return getCCResult(DAG, Intr);
7297}
7298
7299SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op,
7300 SelectionDAG &DAG) const {
7301 SDLoc DL(Op);
7302 SDValue Chain = Op.getOperand(0);
7303
7304 // STCKF only supports a memory operand, so we have to use a temporary.
7305 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
7306 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
7307 MachinePointerInfo MPI =
7309
7310 // Use STCFK to store the TOD clock into the temporary.
7311 SDValue StoreOps[] = {Chain, StackPtr};
7312 Chain = DAG.getMemIntrinsicNode(
7313 SystemZISD::STCKF, DL, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
7314 MPI, MaybeAlign(), MachineMemOperand::MOStore);
7315
7316 // And read it back from there.
7317 return DAG.getLoad(MVT::i64, DL, Chain, StackPtr, MPI);
7318}
7319
7321 SelectionDAG &DAG) const {
7322 switch (Op.getOpcode()) {
7323 case ISD::FRAMEADDR:
7324 return lowerFRAMEADDR(Op, DAG);
7325 case ISD::RETURNADDR:
7326 return lowerRETURNADDR(Op, DAG);
7327 case ISD::BR_CC:
7328 return lowerBR_CC(Op, DAG);
7329 case ISD::SELECT_CC:
7330 return lowerSELECT_CC(Op, DAG);
7331 case ISD::SETCC:
7332 return lowerSETCC(Op, DAG);
7333 case ISD::STRICT_FSETCC:
7334 return lowerSTRICT_FSETCC(Op, DAG, false);
7336 return lowerSTRICT_FSETCC(Op, DAG, true);
7337 case ISD::GlobalAddress:
7338 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG);
7340 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG);
7341 case ISD::BlockAddress:
7342 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG);
7343 case ISD::JumpTable:
7344 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG);
7345 case ISD::ConstantPool:
7346 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG);
7347 case ISD::BITCAST:
7348 return lowerBITCAST(Op, DAG);
7349 case ISD::VASTART:
7350 return lowerVASTART(Op, DAG);
7351 case ISD::VACOPY:
7352 return lowerVACOPY(Op, DAG);
7354 return lowerDYNAMIC_STACKALLOC(Op, DAG);
7356 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
7357 case ISD::MULHS:
7358 return lowerMULH(Op, DAG, SystemZISD::SMUL_LOHI);
7359 case ISD::MULHU:
7360 return lowerMULH(Op, DAG, SystemZISD::UMUL_LOHI);
7361 case ISD::SMUL_LOHI:
7362 return lowerSMUL_LOHI(Op, DAG);
7363 case ISD::UMUL_LOHI:
7364 return lowerUMUL_LOHI(Op, DAG);
7365 case ISD::SDIVREM:
7366 return lowerSDIVREM(Op, DAG);
7367 case ISD::UDIVREM:
7368 return lowerUDIVREM(Op, DAG);
7369 case ISD::SADDO:
7370 case ISD::SSUBO:
7371 case ISD::UADDO:
7372 case ISD::USUBO:
7373 return lowerXALUO(Op, DAG);
7374 case ISD::UADDO_CARRY:
7375 case ISD::USUBO_CARRY:
7376 return lowerUADDSUBO_CARRY(Op, DAG);
7377 case ISD::OR:
7378 return lowerOR(Op, DAG);
7379 case ISD::CTPOP:
7380 return lowerCTPOP(Op, DAG);
7381 case ISD::VECREDUCE_ADD:
7382 return lowerVECREDUCE_ADD(Op, DAG);
7383 case ISD::ATOMIC_FENCE:
7384 return lowerATOMIC_FENCE(Op, DAG);
7385 case ISD::ATOMIC_STORE:
7386 return lowerATOMIC_STORE(Op, DAG);
7387 case ISD::ATOMIC_LOAD:
7388 return lowerATOMIC_LOAD(Op, DAG);
7390 return lowerATOMIC_LOAD_SUB(Op, DAG);
7392 return lowerATOMIC_CMP_SWAP(Op, DAG);
7393 case ISD::STACKSAVE:
7394 return lowerSTACKSAVE(Op, DAG);
7395 case ISD::STACKRESTORE:
7396 return lowerSTACKRESTORE(Op, DAG);
7397 case ISD::PREFETCH:
7398 return lowerPREFETCH(Op, DAG);
7400 return lowerINTRINSIC_W_CHAIN(Op, DAG);
7402 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
7403 case ISD::BUILD_VECTOR:
7404 return lowerBUILD_VECTOR(Op, DAG);
7406 return lowerVECTOR_SHUFFLE(Op, DAG);
7408 return lowerSCALAR_TO_VECTOR(Op, DAG);
7410 return lowerINSERT_VECTOR_ELT(Op, DAG);
7412 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7414 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
7416 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
7417 case ISD::SHL:
7418 return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR);
7419 case ISD::SRL:
7420 return lowerShift(Op, DAG, SystemZISD::VSRL_BY_SCALAR);
7421 case ISD::SRA:
7422 return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR);
7423 case ISD::ADDRSPACECAST:
7424 return lowerAddrSpaceCast(Op, DAG);
7425 case ISD::ROTL:
7426 return lowerShift(Op, DAG, SystemZISD::VROTL_BY_SCALAR);
7427 case ISD::FSHL:
7428 return lowerFSHL(Op, DAG);
7429 case ISD::FSHR:
7430 return lowerFSHR(Op, DAG);
7431 case ISD::FP_EXTEND:
7433 return lowerFP_EXTEND(Op, DAG);
7434 case ISD::FP_TO_UINT:
7435 case ISD::FP_TO_SINT:
7438 return lower_FP_TO_INT(Op, DAG);
7439 case ISD::UINT_TO_FP:
7440 case ISD::SINT_TO_FP:
7443 return lower_INT_TO_FP(Op, DAG);
7444 case ISD::LOAD:
7445 return lowerLoadF16(Op, DAG);
7446 case ISD::STORE:
7447 return lowerStoreF16(Op, DAG);
7448 case ISD::IS_FPCLASS:
7449 return lowerIS_FPCLASS(Op, DAG);
7450 case ISD::GET_ROUNDING:
7451 return lowerGET_ROUNDING(Op, DAG);
7453 return lowerREADCYCLECOUNTER(Op, DAG);
7456 // These operations are legal on our platform, but we cannot actually
7457 // set the operation action to Legal as common code would treat this
7458 // as equivalent to Expand. Instead, we keep the operation action to
7459 // Custom and just leave them unchanged here.
7460 return Op;
7461
7462 default:
7463 llvm_unreachable("Unexpected node to lower");
7464 }
7465}
7466
7468 const SDLoc &SL) {
7469 // If i128 is legal, just use a normal bitcast.
7470 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128))
7471 return DAG.getBitcast(MVT::f128, Src);
7472
7473 // Otherwise, f128 must live in FP128, so do a partwise move.
7475 &SystemZ::FP128BitRegClass);
7476
7477 SDValue Hi, Lo;
7478 std::tie(Lo, Hi) = DAG.SplitScalar(Src, SL, MVT::i64, MVT::i64);
7479
7480 Hi = DAG.getBitcast(MVT::f64, Hi);
7481 Lo = DAG.getBitcast(MVT::f64, Lo);
7482
7483 SDNode *Pair = DAG.getMachineNode(
7484 SystemZ::REG_SEQUENCE, SL, MVT::f128,
7485 {DAG.getTargetConstant(SystemZ::FP128BitRegClassID, SL, MVT::i32), Lo,
7486 DAG.getTargetConstant(SystemZ::subreg_l64, SL, MVT::i32), Hi,
7487 DAG.getTargetConstant(SystemZ::subreg_h64, SL, MVT::i32)});
7488 return SDValue(Pair, 0);
7489}
7490
7492 const SDLoc &SL) {
7493 // If i128 is legal, just use a normal bitcast.
7494 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128))
7495 return DAG.getBitcast(MVT::i128, Src);
7496
7497 // Otherwise, f128 must live in FP128, so do a partwise move.
7499 &SystemZ::FP128BitRegClass);
7500
7501 SDValue LoFP =
7502 DAG.getTargetExtractSubreg(SystemZ::subreg_l64, SL, MVT::f64, Src);
7503 SDValue HiFP =
7504 DAG.getTargetExtractSubreg(SystemZ::subreg_h64, SL, MVT::f64, Src);
7505 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i64, LoFP);
7506 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i64, HiFP);
7507
7508 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i128, Lo, Hi);
7509}
7510
7511// Lower operations with invalid operand or result types.
7512void
7515 SelectionDAG &DAG) const {
7516 switch (N->getOpcode()) {
7517 case ISD::ATOMIC_LOAD: {
7518 SDLoc DL(N);
7519 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
7520 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
7521 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
7522 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_LOAD_128,
7523 DL, Tys, Ops, MVT::i128, MMO);
7524
7525 SDValue Lowered = lowerGR128ToI128(DAG, Res);
7526 if (N->getValueType(0) == MVT::f128)
7527 Lowered = expandBitCastI128ToF128(DAG, Lowered, DL);
7528 Results.push_back(Lowered);
7529 Results.push_back(Res.getValue(1));
7530 break;
7531 }
7532 case ISD::ATOMIC_STORE: {
7533 SDLoc DL(N);
7534 SDVTList Tys = DAG.getVTList(MVT::Other);
7535 SDValue Val = N->getOperand(1);
7536 if (Val.getValueType() == MVT::f128)
7537 Val = expandBitCastF128ToI128(DAG, Val, DL);
7538 Val = lowerI128ToGR128(DAG, Val);
7539
7540 SDValue Ops[] = {N->getOperand(0), Val, N->getOperand(2)};
7541 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
7542 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128,
7543 DL, Tys, Ops, MVT::i128, MMO);
7544 // We have to enforce sequential consistency by performing a
7545 // serialization operation after the store.
7546 if (cast<AtomicSDNode>(N)->getSuccessOrdering() ==
7548 Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL,
7549 MVT::Other, Res), 0);
7550 Results.push_back(Res);
7551 break;
7552 }
7554 SDLoc DL(N);
7555 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other);
7556 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
7557 lowerI128ToGR128(DAG, N->getOperand(2)),
7558 lowerI128ToGR128(DAG, N->getOperand(3)) };
7559 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
7560 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP_128,
7561 DL, Tys, Ops, MVT::i128, MMO);
7562 SDValue Success = emitSETCC(DAG, DL, Res.getValue(1),
7564 Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));
7565 Results.push_back(lowerGR128ToI128(DAG, Res));
7566 Results.push_back(Success);
7567 Results.push_back(Res.getValue(2));
7568 break;
7569 }
7570 case ISD::BITCAST: {
7571 if (useSoftFloat())
7572 return;
7573 SDLoc DL(N);
7574 SDValue Src = N->getOperand(0);
7575 EVT SrcVT = Src.getValueType();
7576 EVT ResVT = N->getValueType(0);
7577 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7578 Results.push_back(expandBitCastF128ToI128(DAG, Src, DL));
7579 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7580 if (Subtarget.hasVector()) {
7581 SDValue In32 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);
7582 Results.push_back(SDValue(
7583 DAG.getMachineNode(SystemZ::LEFR_16, DL, MVT::f16, In32), 0));
7584 } else {
7585 SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Src);
7586 Results.push_back(convertToF16(In64, DAG));
7587 }
7588 } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7589 SDValue ExtractedVal =
7590 Subtarget.hasVector()
7591 ? SDValue(DAG.getMachineNode(SystemZ::LFER_16, DL, MVT::i32, Src),
7592 0)
7593 : convertFromF16(Src, DL, DAG);
7594 Results.push_back(DAG.getZExtOrTrunc(ExtractedVal, DL, ResVT));
7595 }
7596 break;
7597 }
7598 case ISD::UINT_TO_FP:
7599 case ISD::SINT_TO_FP:
7602 if (useSoftFloat())
7603 return;
7604 bool IsStrict = N->isStrictFPOpcode();
7605 SDLoc DL(N);
7606 SDValue InOp = N->getOperand(IsStrict ? 1 : 0);
7607 EVT ResVT = N->getValueType(0);
7608 SDValue Chain = IsStrict ? N->getOperand(0) : DAG.getEntryNode();
7609 if (ResVT == MVT::f16) {
7610 if (!IsStrict) {
7611 SDValue OpF32 = DAG.getNode(N->getOpcode(), DL, MVT::f32, InOp);
7612 Results.push_back(DAG.getFPExtendOrRound(OpF32, DL, MVT::f16));
7613 } else {
7614 SDValue OpF32 =
7615 DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::f32, MVT::Other),
7616 {Chain, InOp});
7617 SDValue F16Res;
7618 std::tie(F16Res, Chain) = DAG.getStrictFPExtendOrRound(
7619 OpF32, OpF32.getValue(1), DL, MVT::f16);
7620 Results.push_back(F16Res);
7621 Results.push_back(Chain);
7622 }
7623 }
7624 break;
7625 }
7626 case ISD::FP_TO_UINT:
7627 case ISD::FP_TO_SINT:
7630 if (useSoftFloat())
7631 return;
7632 bool IsStrict = N->isStrictFPOpcode();
7633 SDLoc DL(N);
7634 EVT ResVT = N->getValueType(0);
7635 SDValue InOp = N->getOperand(IsStrict ? 1 : 0);
7636 EVT InVT = InOp->getValueType(0);
7637 SDValue Chain = IsStrict ? N->getOperand(0) : DAG.getEntryNode();
7638 if (InVT == MVT::f16) {
7639 if (!IsStrict) {
7640 SDValue InF32 = DAG.getFPExtendOrRound(InOp, DL, MVT::f32);
7641 Results.push_back(DAG.getNode(N->getOpcode(), DL, ResVT, InF32));
7642 } else {
7643 SDValue InF32;
7644 std::tie(InF32, Chain) =
7645 DAG.getStrictFPExtendOrRound(InOp, Chain, DL, MVT::f32);
7646 SDValue OpF32 =
7647 DAG.getNode(N->getOpcode(), DL, DAG.getVTList(ResVT, MVT::Other),
7648 {Chain, InF32});
7649 Results.push_back(OpF32);
7650 Results.push_back(OpF32.getValue(1));
7651 }
7652 }
7653 break;
7654 }
7655 case ISD::ATOMIC_SWAP:
7656 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_SWAPW);
7657 break;
7659 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_ADD);
7660 break;
7662 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_SUB);
7663 break;
7665 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_AND);
7666 break;
7668 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_OR);
7669 break;
7671 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_XOR);
7672 break;
7674 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_NAND);
7675 break;
7677 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_MIN);
7678 break;
7680 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_MAX);
7681 break;
7683 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
7684 break;
7686 lowerATOMIC_LOAD_OP(N, Results, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
7687 break;
7688 default:
7689 llvm_unreachable("Unexpected node to lower");
7690 }
7691}
7692
7693void
7699
7700// Return true if VT is a vector whose elements are a whole number of bytes
7701// in width. Also check for presence of vector support.
7702bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
7703 if (!Subtarget.hasVector())
7704 return false;
7705
7706 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
7707}
7708
7709// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
7710// producing a result of type ResVT. Op is a possibly bitcast version
7711// of the input vector and Index is the index (based on type VecVT) that
7712// should be extracted. Return the new extraction if a simplification
7713// was possible or if Force is true.
7714SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
7715 EVT VecVT, SDValue Op,
7716 unsigned Index,
7717 DAGCombinerInfo &DCI,
7718 bool Force) const {
7719 SelectionDAG &DAG = DCI.DAG;
7720
7721 // The number of bytes being extracted.
7722 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7723
7724 for (;;) {
7725 unsigned Opcode = Op.getOpcode();
7726 if (Opcode == ISD::BITCAST)
7727 // Look through bitcasts.
7728 Op = Op.getOperand(0);
7729 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
7730 canTreatAsByteVector(Op.getValueType())) {
7731 // Get a VPERM-like permute mask and see whether the bytes covered
7732 // by the extracted element are a contiguous sequence from one
7733 // source operand.
7735 if (!getVPermMask(Op, Bytes))
7736 break;
7737 int First;
7738 if (!getShuffleInput(Bytes, Index * BytesPerElement,
7739 BytesPerElement, First))
7740 break;
7741 if (First < 0)
7742 return DAG.getUNDEF(ResVT);
7743 // Make sure the contiguous sequence starts at a multiple of the
7744 // original element size.
7745 unsigned Byte = unsigned(First) % Bytes.size();
7746 if (Byte % BytesPerElement != 0)
7747 break;
7748 // We can get the extracted value directly from an input.
7749 Index = Byte / BytesPerElement;
7750 Op = Op.getOperand(unsigned(First) / Bytes.size());
7751 Force = true;
7752 } else if (Opcode == ISD::BUILD_VECTOR &&
7753 canTreatAsByteVector(Op.getValueType())) {
7754 // We can only optimize this case if the BUILD_VECTOR elements are
7755 // at least as wide as the extracted value.
7756 EVT OpVT = Op.getValueType();
7757 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7758 if (OpBytesPerElement < BytesPerElement)
7759 break;
7760 // Make sure that the least-significant bit of the extracted value
7761 // is the least significant bit of an input.
7762 unsigned End = (Index + 1) * BytesPerElement;
7763 if (End % OpBytesPerElement != 0)
7764 break;
7765 // We're extracting the low part of one operand of the BUILD_VECTOR.
7766 Op = Op.getOperand(End / OpBytesPerElement - 1);
7767 EVT ResIntVT = MVT::getIntegerVT(ResVT.getSizeInBits());
7768 if (!isTypeLegal(ResIntVT))
7769 break;
7770 if (!Op.getValueType().isInteger()) {
7771 EVT OpIntVT = MVT::getIntegerVT(Op.getValueSizeInBits());
7772 if (!isTypeLegal(OpIntVT))
7773 break;
7774 Op = DAG.getNode(ISD::BITCAST, DL, OpIntVT, Op);
7775 DCI.AddToWorklist(Op.getNode());
7776 }
7777 Op = DAG.getNode(ISD::TRUNCATE, DL, ResIntVT, Op);
7778 if (ResIntVT != ResVT) {
7779 DCI.AddToWorklist(Op.getNode());
7780 Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op);
7781 }
7782 return Op;
7783 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
7785 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
7786 canTreatAsByteVector(Op.getValueType()) &&
7787 canTreatAsByteVector(Op.getOperand(0).getValueType())) {
7788 // Make sure that only the unextended bits are significant.
7789 EVT ExtVT = Op.getValueType();
7790 EVT OpVT = Op.getOperand(0).getValueType();
7791 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
7792 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7793 unsigned Byte = Index * BytesPerElement;
7794 unsigned SubByte = Byte % ExtBytesPerElement;
7795 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7796 if (SubByte < MinSubByte ||
7797 SubByte + BytesPerElement > ExtBytesPerElement)
7798 break;
7799 // Get the byte offset of the unextended element
7800 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
7801 // ...then add the byte offset relative to that element.
7802 Byte += SubByte - MinSubByte;
7803 if (Byte % BytesPerElement != 0)
7804 break;
7805 Op = Op.getOperand(0);
7806 Index = Byte / BytesPerElement;
7807 Force = true;
7808 } else
7809 break;
7810 }
7811 if (Force) {
7812 if (Op.getValueType() != VecVT) {
7813 Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op);
7814 DCI.AddToWorklist(Op.getNode());
7815 }
7816 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op,
7817 DAG.getConstant(Index, DL, MVT::i32));
7818 }
7819 return SDValue();
7820}
7821
7822// Optimize vector operations in scalar value Op on the basis that Op
7823// is truncated to TruncVT.
7824SDValue SystemZTargetLowering::combineTruncateExtract(
7825 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
7826 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
7827 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
7828 // of type TruncVT.
7829 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7830 TruncVT.getSizeInBits() % 8 == 0) {
7831 SDValue Vec = Op.getOperand(0);
7832 EVT VecVT = Vec.getValueType();
7833 if (canTreatAsByteVector(VecVT)) {
7834 if (auto *IndexN = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
7835 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7836 unsigned TruncBytes = TruncVT.getStoreSize();
7837 if (BytesPerElement % TruncBytes == 0) {
7838 // Calculate the value of Y' in the above description. We are
7839 // splitting the original elements into Scale equal-sized pieces
7840 // and for truncation purposes want the last (least-significant)
7841 // of these pieces for IndexN. This is easiest to do by calculating
7842 // the start index of the following element and then subtracting 1.
7843 unsigned Scale = BytesPerElement / TruncBytes;
7844 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7845
7846 // Defer the creation of the bitcast from X to combineExtract,
7847 // which might be able to optimize the extraction.
7848 VecVT = EVT::getVectorVT(*DCI.DAG.getContext(),
7849 MVT::getIntegerVT(TruncBytes * 8),
7850 VecVT.getStoreSize() / TruncBytes);
7851 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7852 return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true);
7853 }
7854 }
7855 }
7856 }
7857 return SDValue();
7858}
7859
7860SDValue SystemZTargetLowering::combineZERO_EXTEND(
7861 SDNode *N, DAGCombinerInfo &DCI) const {
7862 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
7863 SelectionDAG &DAG = DCI.DAG;
7864 SDValue N0 = N->getOperand(0);
7865 EVT VT = N->getValueType(0);
7866 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
7867 auto *TrueOp = dyn_cast<ConstantSDNode>(N0.getOperand(0));
7868 auto *FalseOp = dyn_cast<ConstantSDNode>(N0.getOperand(1));
7869 if (TrueOp && FalseOp) {
7870 SDLoc DL(N0);
7871 SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),
7872 DAG.getConstant(FalseOp->getZExtValue(), DL, VT),
7873 N0.getOperand(2), N0.getOperand(3), N0.getOperand(4) };
7874 SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7875 // If N0 has multiple uses, change other uses as well.
7876 if (!N0.hasOneUse()) {
7877 SDValue TruncSelect =
7878 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), NewSelect);
7879 DCI.CombineTo(N0.getNode(), TruncSelect);
7880 }
7881 return NewSelect;
7882 }
7883 }
7884 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
7885 // of the result is smaller than the size of X and all the truncated bits
7886 // of X are already zero.
7887 if (N0.getOpcode() == ISD::XOR &&
7888 N0.hasOneUse() && N0.getOperand(0).hasOneUse() &&
7889 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
7890 N0.getOperand(1).getOpcode() == ISD::Constant) {
7891 SDValue X = N0.getOperand(0).getOperand(0);
7892 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
7893 KnownBits Known = DAG.computeKnownBits(X);
7894 APInt TruncatedBits = APInt::getBitsSet(X.getValueSizeInBits(),
7895 N0.getValueSizeInBits(),
7896 VT.getSizeInBits());
7897 if (TruncatedBits.isSubsetOf(Known.Zero)) {
7898 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
7899 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
7900 return DAG.getNode(ISD::XOR, SDLoc(N0), VT,
7901 X, DAG.getConstant(Mask, SDLoc(N0), VT));
7902 }
7903 }
7904 }
7905 // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
7906 // and VECTOR ADD COMPUTE CARRY for i128:
7907 // (zext (setcc_uge X Y)) --> (VSCBI X Y)
7908 // (zext (setcc_ule Y X)) --> (VSCBI X Y)
7909 // (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
7910 // (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
7911 // For vector types, these patterns are recognized in the .td file.
7912 if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
7913 N0.getOperand(0).getValueType() == VT) {
7914 SDValue Op0 = N0.getOperand(0);
7915 SDValue Op1 = N0.getOperand(1);
7916 const ISD::CondCode CC = N0.getOperandCondCode(2);
7917 switch (CC) {
7918 case ISD::SETULE:
7919 std::swap(Op0, Op1);
7920 [[fallthrough]];
7921 case ISD::SETUGE:
7922 return DAG.getNode(SystemZISD::VSCBI, SDLoc(N0), VT, Op0, Op1);
7923 case ISD::SETUGT:
7924 std::swap(Op0, Op1);
7925 [[fallthrough]];
7926 case ISD::SETULT:
7927 if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
7928 (Op0->getOperand(0) == Op1 || Op0->getOperand(1) == Op1))
7929 return DAG.getNode(SystemZISD::VACC, SDLoc(N0), VT, Op0->getOperand(0),
7930 Op0->getOperand(1));
7931 break;
7932 default:
7933 break;
7934 }
7935 }
7936
7937 return SDValue();
7938}
7939
7940SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7941 SDNode *N, DAGCombinerInfo &DCI) const {
7942 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
7943 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
7944 // into (select_cc LHS, RHS, -1, 0, COND)
7945 SelectionDAG &DAG = DCI.DAG;
7946 SDValue N0 = N->getOperand(0);
7947 EVT VT = N->getValueType(0);
7948 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
7949 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
7950 N0 = N0.getOperand(0);
7951 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
7952 SDLoc DL(N0);
7953 SDValue Ops[] = { N0.getOperand(0), N0.getOperand(1),
7954 DAG.getAllOnesConstant(DL, VT),
7955 DAG.getConstant(0, DL, VT), N0.getOperand(2) };
7956 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
7957 }
7958 return SDValue();
7959}
7960
7961SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7962 SDNode *N, DAGCombinerInfo &DCI) const {
7963 // Convert (sext (ashr (shl X, C1), C2)) to
7964 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
7965 // cheap as narrower ones.
7966 SelectionDAG &DAG = DCI.DAG;
7967 SDValue N0 = N->getOperand(0);
7968 EVT VT = N->getValueType(0);
7969 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
7970 auto *SraAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
7971 SDValue Inner = N0.getOperand(0);
7972 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
7973 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.getOperand(1))) {
7974 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
7975 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7976 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7977 EVT ShiftVT = N0.getOperand(1).getValueType();
7978 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT,
7979 Inner.getOperand(0));
7980 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext,
7981 DAG.getConstant(NewShlAmt, SDLoc(Inner),
7982 ShiftVT));
7983 return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl,
7984 DAG.getConstant(NewSraAmt, SDLoc(N0), ShiftVT));
7985 }
7986 }
7987 }
7988
7989 return SDValue();
7990}
7991
7992SDValue SystemZTargetLowering::combineMERGE(
7993 SDNode *N, DAGCombinerInfo &DCI) const {
7994 SelectionDAG &DAG = DCI.DAG;
7995 unsigned Opcode = N->getOpcode();
7996 SDValue Op0 = N->getOperand(0);
7997 SDValue Op1 = N->getOperand(1);
7998 if (Op0.getOpcode() == ISD::BITCAST)
7999 Op0 = Op0.getOperand(0);
8000 if (ISD::isBuildVectorAllZeros(Op0)) {
8001 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
8002 // for v4f32.
8003 if (Op1 == N->getOperand(0))
8004 return Op1;
8005 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
8006 EVT VT = Op1.getValueType();
8007 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
8008 if (ElemBytes <= 4) {
8009 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
8010 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
8011 EVT InVT = VT.changeVectorElementTypeToInteger();
8012 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(ElemBytes * 16),
8013 SystemZ::VectorBytes / ElemBytes / 2);
8014 if (VT != InVT) {
8015 Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), InVT, Op1);
8016 DCI.AddToWorklist(Op1.getNode());
8017 }
8018 SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1);
8019 DCI.AddToWorklist(Op.getNode());
8020 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
8021 }
8022 }
8023 return SDValue();
8024}
8025
8026static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
8027 SDNode *&HiPart) {
8028 LoPart = HiPart = nullptr;
8029
8030 // Scan through all users.
8031 for (SDUse &Use : LD->uses()) {
8032 // Skip the uses of the chain.
8033 if (Use.getResNo() != 0)
8034 continue;
8035
8036 // Verify every user is a TRUNCATE to i64 of the low or high half.
8037 SDNode *User = Use.getUser();
8038 bool IsLoPart = true;
8039 if (User->getOpcode() == ISD::SRL &&
8040 User->getOperand(1).getOpcode() == ISD::Constant &&
8041 User->getConstantOperandVal(1) == 64 && User->hasOneUse()) {
8042 User = *User->user_begin();
8043 IsLoPart = false;
8044 }
8045 if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(0) != MVT::i64)
8046 return false;
8047
8048 if (IsLoPart) {
8049 if (LoPart)
8050 return false;
8051 LoPart = User;
8052 } else {
8053 if (HiPart)
8054 return false;
8055 HiPart = User;
8056 }
8057 }
8058 return true;
8059}
8060
8061static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
8062 SDNode *&HiPart) {
8063 LoPart = HiPart = nullptr;
8064
8065 // Scan through all users.
8066 for (SDUse &Use : LD->uses()) {
8067 // Skip the uses of the chain.
8068 if (Use.getResNo() != 0)
8069 continue;
8070
8071 // Verify every user is an EXTRACT_SUBREG of the low or high half.
8072 SDNode *User = Use.getUser();
8073 if (!User->hasOneUse() || !User->isMachineOpcode() ||
8074 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
8075 return false;
8076
8077 switch (User->getConstantOperandVal(1)) {
8078 case SystemZ::subreg_l64:
8079 if (LoPart)
8080 return false;
8081 LoPart = User;
8082 break;
8083 case SystemZ::subreg_h64:
8084 if (HiPart)
8085 return false;
8086 HiPart = User;
8087 break;
8088 default:
8089 return false;
8090 }
8091 }
8092 return true;
8093}
8094
8095SDValue SystemZTargetLowering::combineLOAD(
8096 SDNode *N, DAGCombinerInfo &DCI) const {
8097 SelectionDAG &DAG = DCI.DAG;
8098 EVT LdVT = N->getValueType(0);
8099 if (auto *LN = dyn_cast<LoadSDNode>(N)) {
8100 if (LN->getAddressSpace() == SYSTEMZAS::PTR32) {
8101 MVT PtrVT = getPointerTy(DAG.getDataLayout());
8102 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
8103 if (PtrVT != LoadNodeVT) {
8104 SDLoc DL(LN);
8105 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
8106 DL, PtrVT, LN->getBasePtr(), SYSTEMZAS::PTR32, 0);
8107 return DAG.getExtLoad(LN->getExtensionType(), DL, LN->getValueType(0),
8108 LN->getChain(), AddrSpaceCast, LN->getMemoryVT(),
8109 LN->getMemOperand());
8110 }
8111 }
8112 }
8113 SDLoc DL(N);
8114
8115 // Replace a 128-bit load that is used solely to move its value into GPRs
8116 // by separate loads of both halves.
8117 LoadSDNode *LD = cast<LoadSDNode>(N);
8118 if (LD->isSimple() && ISD::isNormalLoad(LD)) {
8119 SDNode *LoPart, *HiPart;
8120 if ((LdVT == MVT::i128 && isI128MovedToParts(LD, LoPart, HiPart)) ||
8121 (LdVT == MVT::f128 && isF128MovedToParts(LD, LoPart, HiPart))) {
8122 // Rewrite each extraction as an independent load.
8123 SmallVector<SDValue, 2> ArgChains;
8124 if (HiPart) {
8125 SDValue EltLoad = DAG.getLoad(
8126 HiPart->getValueType(0), DL, LD->getChain(), LD->getBasePtr(),
8127 LD->getPointerInfo(), LD->getBaseAlign(),
8128 LD->getMemOperand()->getFlags(), LD->getAAInfo());
8129
8130 DCI.CombineTo(HiPart, EltLoad, true);
8131 ArgChains.push_back(EltLoad.getValue(1));
8132 }
8133 if (LoPart) {
8134 SDValue EltLoad = DAG.getLoad(
8135 LoPart->getValueType(0), DL, LD->getChain(),
8136 DAG.getObjectPtrOffset(DL, LD->getBasePtr(), TypeSize::getFixed(8)),
8137 LD->getPointerInfo().getWithOffset(8), LD->getBaseAlign(),
8138 LD->getMemOperand()->getFlags(), LD->getAAInfo());
8139
8140 DCI.CombineTo(LoPart, EltLoad, true);
8141 ArgChains.push_back(EltLoad.getValue(1));
8142 }
8143
8144 // Collect all chains via TokenFactor.
8145 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, ArgChains);
8146 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
8147 DCI.AddToWorklist(Chain.getNode());
8148 return SDValue(N, 0);
8149 }
8150 }
8151
8152 if (LdVT.isVector() || LdVT.isInteger())
8153 return SDValue();
8154 // Transform a scalar load that is REPLICATEd as well as having other
8155 // use(s) to the form where the other use(s) use the first element of the
8156 // REPLICATE instead of the load. Otherwise instruction selection will not
8157 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
8158 // point loads.
8159
8160 SDValue Replicate;
8161 SmallVector<SDNode*, 8> OtherUses;
8162 for (SDUse &Use : N->uses()) {
8163 if (Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
8164 if (Replicate)
8165 return SDValue(); // Should never happen
8166 Replicate = SDValue(Use.getUser(), 0);
8167 } else if (Use.getResNo() == 0)
8168 OtherUses.push_back(Use.getUser());
8169 }
8170 if (!Replicate || OtherUses.empty())
8171 return SDValue();
8172
8173 SDValue Extract0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, LdVT,
8174 Replicate, DAG.getConstant(0, DL, MVT::i32));
8175 // Update uses of the loaded Value while preserving old chains.
8176 for (SDNode *U : OtherUses) {
8178 for (SDValue Op : U->ops())
8179 Ops.push_back((Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
8180 DAG.UpdateNodeOperands(U, Ops);
8181 }
8182 return SDValue(N, 0);
8183}
8184
8185bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
8186 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
8187 return true;
8188 if (Subtarget.hasVectorEnhancements2())
8189 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
8190 return true;
8191 return false;
8192}
8193
8195 if (!VT.isVector() || !VT.isSimple() ||
8196 VT.getSizeInBits() != 128 ||
8197 VT.getScalarSizeInBits() % 8 != 0)
8198 return false;
8199
8200 unsigned NumElts = VT.getVectorNumElements();
8201 for (unsigned i = 0; i < NumElts; ++i) {
8202 if (M[i] < 0) continue; // ignore UNDEF indices
8203 if ((unsigned) M[i] != NumElts - 1 - i)
8204 return false;
8205 }
8206
8207 return true;
8208}
8209
8210static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
8211 for (auto *U : StoredVal->users()) {
8212 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(U)) {
8213 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8214 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
8215 continue;
8216 } else if (isa<BuildVectorSDNode>(U)) {
8217 SDValue BuildVector = SDValue(U, 0);
8218 if (DAG.isSplatValue(BuildVector, true/*AllowUndefs*/) &&
8219 isOnlyUsedByStores(BuildVector, DAG))
8220 continue;
8221 }
8222 return false;
8223 }
8224 return true;
8225}
8226
8227static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart,
8228 SDValue &HiPart) {
8229 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
8230 return false;
8231
8232 SDValue Op0 = Val.getOperand(0);
8233 SDValue Op1 = Val.getOperand(1);
8234
8235 if (Op0.getOpcode() == ISD::SHL)
8236 std::swap(Op0, Op1);
8237 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
8238 Op1.getOperand(1).getOpcode() != ISD::Constant ||
8239 Op1.getConstantOperandVal(1) != 64)
8240 return false;
8241 Op1 = Op1.getOperand(0);
8242
8243 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
8244 Op0.getOperand(0).getValueType() != MVT::i64)
8245 return false;
8246 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
8247 Op1.getOperand(0).getValueType() != MVT::i64)
8248 return false;
8249
8250 LoPart = Op0.getOperand(0);
8251 HiPart = Op1.getOperand(0);
8252 return true;
8253}
8254
8255static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart,
8256 SDValue &HiPart) {
8257 if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||
8258 Val.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
8259 return false;
8260
8261 if (Val->getNumOperands() != 5 ||
8262 Val->getOperand(0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||
8263 Val->getOperand(2)->getAsZExtVal() != SystemZ::subreg_l64 ||
8264 Val->getOperand(4)->getAsZExtVal() != SystemZ::subreg_h64)
8265 return false;
8266
8267 LoPart = Val->getOperand(1);
8268 HiPart = Val->getOperand(3);
8269 return true;
8270}
8271
8272SDValue SystemZTargetLowering::combineSTORE(
8273 SDNode *N, DAGCombinerInfo &DCI) const {
8274 SelectionDAG &DAG = DCI.DAG;
8275 auto *SN = cast<StoreSDNode>(N);
8276 auto &Op1 = N->getOperand(1);
8277 EVT MemVT = SN->getMemoryVT();
8278
8279 if (SN->getAddressSpace() == SYSTEMZAS::PTR32) {
8280 MVT PtrVT = getPointerTy(DAG.getDataLayout());
8281 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8282 if (PtrVT != StoreNodeVT) {
8283 SDLoc DL(SN);
8284 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(DL, PtrVT, SN->getBasePtr(),
8285 SYSTEMZAS::PTR32, 0);
8286 return DAG.getStore(SN->getChain(), DL, SN->getValue(), AddrSpaceCast,
8287 SN->getPointerInfo(), SN->getBaseAlign(),
8288 SN->getMemOperand()->getFlags(), SN->getAAInfo());
8289 }
8290 }
8291
8292 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
8293 // for the extraction to be done on a vMiN value, so that we can use VSTE.
8294 // If X has wider elements then convert it to:
8295 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
8296 if (MemVT.isInteger() && SN->isTruncatingStore()) {
8297 if (SDValue Value =
8298 combineTruncateExtract(SDLoc(N), MemVT, SN->getValue(), DCI)) {
8299 DCI.AddToWorklist(Value.getNode());
8300
8301 // Rewrite the store with the new form of stored value.
8302 return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value,
8303 SN->getBasePtr(), SN->getMemoryVT(),
8304 SN->getMemOperand());
8305 }
8306 }
8307
8308 // combine STORE (LOAD_STACK_GUARD) into MOV_STACKGUARD_DAG
8309 if (Op1->isMachineOpcode() &&
8310 (Op1->getMachineOpcode() == SystemZ::LOAD_STACK_GUARD)) {
8311 // Obtain the frame index the store was targeting.
8312 int FI = cast<FrameIndexSDNode>(SN->getOperand(2))->getIndex();
8313 // Prepare operands of the MOV_STACKGUARD ISD Node - Chain and FrameIndex.
8314 SDValue Ops[] = {SN->getChain(), DAG.getTargetFrameIndex(FI, MVT::i64)};
8315 return DAG.getNode(SystemZISD::MOV_STACKGUARD, SDLoc(SN), MVT::Other, Ops);
8316 }
8317
8318 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
8319 if (!SN->isTruncatingStore() &&
8320 Op1.getOpcode() == ISD::BSWAP &&
8321 Op1.getNode()->hasOneUse() &&
8322 canLoadStoreByteSwapped(Op1.getValueType())) {
8323
8324 SDValue BSwapOp = Op1.getOperand(0);
8325
8326 if (BSwapOp.getValueType() == MVT::i16)
8327 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), MVT::i32, BSwapOp);
8328
8329 SDValue Ops[] = {
8330 N->getOperand(0), BSwapOp, N->getOperand(2)
8331 };
8332
8333 return
8334 DAG.getMemIntrinsicNode(SystemZISD::STRV, SDLoc(N), DAG.getVTList(MVT::Other),
8335 Ops, MemVT, SN->getMemOperand());
8336 }
8337 // Combine STORE (element-swap) into VSTER
8338 if (!SN->isTruncatingStore() &&
8339 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
8340 Op1.getNode()->hasOneUse() &&
8341 Subtarget.hasVectorEnhancements2()) {
8342 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op1.getNode());
8343 ArrayRef<int> ShuffleMask = SVN->getMask();
8344 if (isVectorElementSwap(ShuffleMask, Op1.getValueType())) {
8345 SDValue Ops[] = {
8346 N->getOperand(0), Op1.getOperand(0), N->getOperand(2)
8347 };
8348
8349 return DAG.getMemIntrinsicNode(SystemZISD::VSTER, SDLoc(N),
8350 DAG.getVTList(MVT::Other),
8351 Ops, MemVT, SN->getMemOperand());
8352 }
8353 }
8354
8355 // Combine STORE (READCYCLECOUNTER) into STCKF.
8356 if (!SN->isTruncatingStore() &&
8358 Op1.hasOneUse() &&
8359 N->getOperand(0).reachesChainWithoutSideEffects(SDValue(Op1.getNode(), 1))) {
8360 SDValue Ops[] = { Op1.getOperand(0), N->getOperand(2) };
8361 return DAG.getMemIntrinsicNode(SystemZISD::STCKF, SDLoc(N),
8362 DAG.getVTList(MVT::Other),
8363 Ops, MemVT, SN->getMemOperand());
8364 }
8365
8366 // Transform a store of a 128-bit value moved from parts into two stores.
8367 if (SN->isSimple() && ISD::isNormalStore(SN)) {
8368 SDValue LoPart, HiPart;
8369 if ((MemVT == MVT::i128 && isI128MovedFromParts(Op1, LoPart, HiPart)) ||
8370 (MemVT == MVT::f128 && isF128MovedFromParts(Op1, LoPart, HiPart))) {
8371 SDLoc DL(SN);
8372 SDValue Chain0 = DAG.getStore(
8373 SN->getChain(), DL, HiPart, SN->getBasePtr(), SN->getPointerInfo(),
8374 SN->getBaseAlign(), SN->getMemOperand()->getFlags(), SN->getAAInfo());
8375 SDValue Chain1 = DAG.getStore(
8376 SN->getChain(), DL, LoPart,
8377 DAG.getObjectPtrOffset(DL, SN->getBasePtr(), TypeSize::getFixed(8)),
8378 SN->getPointerInfo().getWithOffset(8), SN->getBaseAlign(),
8379 SN->getMemOperand()->getFlags(), SN->getAAInfo());
8380
8381 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain0, Chain1);
8382 }
8383 }
8384
8385 // Replicate a reg or immediate with VREP instead of scalar multiply or
8386 // immediate load. It seems best to do this during the first DAGCombine as
8387 // it is straight-forward to handle the zero-extend node in the initial
8388 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
8389 // extracting an i16 element from a v16i8 vector).
8390 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
8391 isOnlyUsedByStores(Op1, DAG)) {
8392 SDValue Word = SDValue();
8393 EVT WordVT;
8394
8395 // Find a replicated immediate and return it if found in Word and its
8396 // type in WordVT.
8397 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
8398 // Some constants are better handled with a scalar store.
8399 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
8400 isInt<16>(C->getSExtValue()) || MemVT.getStoreSize() <= 2)
8401 return;
8402
8403 APInt Val = C->getAPIntValue();
8404 // Truncate Val in case of a truncating store.
8405 if (!llvm::isUIntN(TotBytes * 8, Val.getZExtValue())) {
8406 assert(SN->isTruncatingStore() &&
8407 "Non-truncating store and immediate value does not fit?");
8408 Val = Val.trunc(TotBytes * 8);
8409 }
8410
8411 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.getZExtValue()));
8412 if (VCI.isVectorConstantLegal(Subtarget) &&
8413 VCI.Opcode == SystemZISD::REPLICATE) {
8414 Word = DAG.getConstant(VCI.OpVals[0], SDLoc(SN), MVT::i32);
8415 WordVT = VCI.VecVT.getScalarType();
8416 }
8417 };
8418
8419 // Find a replicated register and return it if found in Word and its type
8420 // in WordVT.
8421 auto FindReplicatedReg = [&](SDValue MulOp) {
8422 EVT MulVT = MulOp.getValueType();
8423 if (MulOp->getOpcode() == ISD::MUL &&
8424 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8425 // Find a zero extended value and its type.
8426 SDValue LHS = MulOp->getOperand(0);
8427 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
8428 WordVT = LHS->getOperand(0).getValueType();
8429 else if (LHS->getOpcode() == ISD::AssertZext)
8430 WordVT = cast<VTSDNode>(LHS->getOperand(1))->getVT();
8431 else
8432 return;
8433 // Find a replicating constant, e.g. 0x00010001.
8434 if (auto *C = dyn_cast<ConstantSDNode>(MulOp->getOperand(1))) {
8435 SystemZVectorConstantInfo VCI(
8436 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
8437 if (VCI.isVectorConstantLegal(Subtarget) &&
8438 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8439 WordVT == VCI.VecVT.getScalarType())
8440 Word = DAG.getZExtOrTrunc(LHS->getOperand(0), SDLoc(SN), WordVT);
8441 }
8442 }
8443 };
8444
8445 if (isa<BuildVectorSDNode>(Op1) &&
8446 DAG.isSplatValue(Op1, true/*AllowUndefs*/)) {
8447 SDValue SplatVal = Op1->getOperand(0);
8448 if (auto *C = dyn_cast<ConstantSDNode>(SplatVal))
8449 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
8450 else
8451 FindReplicatedReg(SplatVal);
8452 } else {
8453 if (auto *C = dyn_cast<ConstantSDNode>(Op1))
8454 FindReplicatedImm(C, MemVT.getStoreSize());
8455 else
8456 FindReplicatedReg(Op1);
8457 }
8458
8459 if (Word != SDValue()) {
8460 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
8461 "Bad type handling");
8462 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
8463 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), WordVT, NumElts);
8464 SDValue SplatVal = DAG.getSplatVector(SplatVT, SDLoc(SN), Word);
8465 return DAG.getStore(SN->getChain(), SDLoc(SN), SplatVal,
8466 SN->getBasePtr(), SN->getMemOperand());
8467 }
8468 }
8469
8470 return SDValue();
8471}
8472
8473SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8474 SDNode *N, DAGCombinerInfo &DCI) const {
8475 SelectionDAG &DAG = DCI.DAG;
8476 // Combine element-swap (LOAD) into VLER
8477 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
8478 N->getOperand(0).hasOneUse() &&
8479 Subtarget.hasVectorEnhancements2()) {
8480 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
8481 ArrayRef<int> ShuffleMask = SVN->getMask();
8482 if (isVectorElementSwap(ShuffleMask, N->getValueType(0))) {
8483 SDValue Load = N->getOperand(0);
8484 LoadSDNode *LD = cast<LoadSDNode>(Load);
8485
8486 // Create the element-swapping load.
8487 SDValue Ops[] = {
8488 LD->getChain(), // Chain
8489 LD->getBasePtr() // Ptr
8490 };
8491 SDValue ESLoad =
8492 DAG.getMemIntrinsicNode(SystemZISD::VLER, SDLoc(N),
8493 DAG.getVTList(LD->getValueType(0), MVT::Other),
8494 Ops, LD->getMemoryVT(), LD->getMemOperand());
8495
8496 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
8497 // by the load dead.
8498 DCI.CombineTo(N, ESLoad);
8499
8500 // Next, combine the load away, we give it a bogus result value but a real
8501 // chain result. The result value is dead because the shuffle is dead.
8502 DCI.CombineTo(Load.getNode(), ESLoad, ESLoad.getValue(1));
8503
8504 // Return N so it doesn't get rechecked!
8505 return SDValue(N, 0);
8506 }
8507 }
8508
8509 return SDValue();
8510}
8511
8512SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8513 SDNode *N, DAGCombinerInfo &DCI) const {
8514 SelectionDAG &DAG = DCI.DAG;
8515
8516 if (!Subtarget.hasVector())
8517 return SDValue();
8518
8519 // Look through bitcasts that retain the number of vector elements.
8520 SDValue Op = N->getOperand(0);
8521 if (Op.getOpcode() == ISD::BITCAST &&
8522 Op.getValueType().isVector() &&
8523 Op.getOperand(0).getValueType().isVector() &&
8524 Op.getValueType().getVectorNumElements() ==
8525 Op.getOperand(0).getValueType().getVectorNumElements())
8526 Op = Op.getOperand(0);
8527
8528 // Pull BSWAP out of a vector extraction.
8529 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
8530 EVT VecVT = Op.getValueType();
8531 EVT EltVT = VecVT.getVectorElementType();
8532 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), EltVT,
8533 Op.getOperand(0), N->getOperand(1));
8534 DCI.AddToWorklist(Op.getNode());
8535 Op = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Op);
8536 if (EltVT != N->getValueType(0)) {
8537 DCI.AddToWorklist(Op.getNode());
8538 Op = DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op);
8539 }
8540 return Op;
8541 }
8542
8543 // Try to simplify a vector extraction.
8544 if (auto *IndexN = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
8545 SDValue Op0 = N->getOperand(0);
8546 EVT VecVT = Op0.getValueType();
8547 if (canTreatAsByteVector(VecVT))
8548 return combineExtract(SDLoc(N), N->getValueType(0), VecVT, Op0,
8549 IndexN->getZExtValue(), DCI, false);
8550 }
8551 return SDValue();
8552}
8553
8554SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8555 SDNode *N, DAGCombinerInfo &DCI) const {
8556 SelectionDAG &DAG = DCI.DAG;
8557 // (join_dwords X, X) == (replicate X)
8558 if (N->getOperand(0) == N->getOperand(1))
8559 return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0),
8560 N->getOperand(0));
8561 return SDValue();
8562}
8563
8565 SDValue Chain1 = N1->getOperand(0);
8566 SDValue Chain2 = N2->getOperand(0);
8567
8568 // Trivial case: both nodes take the same chain.
8569 if (Chain1 == Chain2)
8570 return Chain1;
8571
8572 // FIXME - we could handle more complex cases via TokenFactor,
8573 // assuming we can verify that this would not create a cycle.
8574 return SDValue();
8575}
8576
8577SDValue SystemZTargetLowering::combineFP_ROUND(
8578 SDNode *N, DAGCombinerInfo &DCI) const {
8579
8580 if (!Subtarget.hasVector())
8581 return SDValue();
8582
8583 // (fpround (extract_vector_elt X 0))
8584 // (fpround (extract_vector_elt X 1)) ->
8585 // (extract_vector_elt (VROUND X) 0)
8586 // (extract_vector_elt (VROUND X) 2)
8587 //
8588 // This is a special case since the target doesn't really support v2f32s.
8589 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8590 SelectionDAG &DAG = DCI.DAG;
8591 SDValue Op0 = N->getOperand(OpNo);
8592 if (N->getValueType(0) == MVT::f32 && Op0.hasOneUse() &&
8594 Op0.getOperand(0).getValueType() == MVT::v2f64 &&
8595 Op0.getOperand(1).getOpcode() == ISD::Constant &&
8596 Op0.getConstantOperandVal(1) == 0) {
8597 SDValue Vec = Op0.getOperand(0);
8598 for (auto *U : Vec->users()) {
8599 if (U != Op0.getNode() && U->hasOneUse() &&
8600 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8601 U->getOperand(0) == Vec &&
8602 U->getOperand(1).getOpcode() == ISD::Constant &&
8603 U->getConstantOperandVal(1) == 1) {
8604 SDValue OtherRound = SDValue(*U->user_begin(), 0);
8605 if (OtherRound.getOpcode() == N->getOpcode() &&
8606 OtherRound.getOperand(OpNo) == SDValue(U, 0) &&
8607 OtherRound.getValueType() == MVT::f32) {
8608 SDValue VRound, Chain;
8609 if (N->isStrictFPOpcode()) {
8610 Chain = MergeInputChains(N, OtherRound.getNode());
8611 if (!Chain)
8612 continue;
8613 VRound = DAG.getNode(SystemZISD::STRICT_VROUND, SDLoc(N),
8614 {MVT::v4f32, MVT::Other}, {Chain, Vec});
8615 Chain = VRound.getValue(1);
8616 } else
8617 VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N),
8618 MVT::v4f32, Vec);
8619 DCI.AddToWorklist(VRound.getNode());
8620 SDValue Extract1 =
8621 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f32,
8622 VRound, DAG.getConstant(2, SDLoc(U), MVT::i32));
8623 DCI.AddToWorklist(Extract1.getNode());
8624 DAG.ReplaceAllUsesOfValueWith(OtherRound, Extract1);
8625 if (Chain)
8626 DAG.ReplaceAllUsesOfValueWith(OtherRound.getValue(1), Chain);
8627 SDValue Extract0 =
8628 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f32,
8629 VRound, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
8630 if (Chain)
8631 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
8632 N->getVTList(), Extract0, Chain);
8633 return Extract0;
8634 }
8635 }
8636 }
8637 }
8638 return SDValue();
8639}
8640
8641SDValue SystemZTargetLowering::combineFP_EXTEND(
8642 SDNode *N, DAGCombinerInfo &DCI) const {
8643
8644 if (!Subtarget.hasVector())
8645 return SDValue();
8646
8647 // (fpextend (extract_vector_elt X 0))
8648 // (fpextend (extract_vector_elt X 2)) ->
8649 // (extract_vector_elt (VEXTEND X) 0)
8650 // (extract_vector_elt (VEXTEND X) 1)
8651 //
8652 // This is a special case since the target doesn't really support v2f32s.
8653 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8654 SelectionDAG &DAG = DCI.DAG;
8655 SDValue Op0 = N->getOperand(OpNo);
8656 if (N->getValueType(0) == MVT::f64 && Op0.hasOneUse() &&
8658 Op0.getOperand(0).getValueType() == MVT::v4f32 &&
8659 Op0.getOperand(1).getOpcode() == ISD::Constant &&
8660 Op0.getConstantOperandVal(1) == 0) {
8661 SDValue Vec = Op0.getOperand(0);
8662 for (auto *U : Vec->users()) {
8663 if (U != Op0.getNode() && U->hasOneUse() &&
8664 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8665 U->getOperand(0) == Vec &&
8666 U->getOperand(1).getOpcode() == ISD::Constant &&
8667 U->getConstantOperandVal(1) == 2) {
8668 SDValue OtherExtend = SDValue(*U->user_begin(), 0);
8669 if (OtherExtend.getOpcode() == N->getOpcode() &&
8670 OtherExtend.getOperand(OpNo) == SDValue(U, 0) &&
8671 OtherExtend.getValueType() == MVT::f64) {
8672 SDValue VExtend, Chain;
8673 if (N->isStrictFPOpcode()) {
8674 Chain = MergeInputChains(N, OtherExtend.getNode());
8675 if (!Chain)
8676 continue;
8677 VExtend = DAG.getNode(SystemZISD::STRICT_VEXTEND, SDLoc(N),
8678 {MVT::v2f64, MVT::Other}, {Chain, Vec});
8679 Chain = VExtend.getValue(1);
8680 } else
8681 VExtend = DAG.getNode(SystemZISD::VEXTEND, SDLoc(N),
8682 MVT::v2f64, Vec);
8683 DCI.AddToWorklist(VExtend.getNode());
8684 SDValue Extract1 =
8685 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f64,
8686 VExtend, DAG.getConstant(1, SDLoc(U), MVT::i32));
8687 DCI.AddToWorklist(Extract1.getNode());
8688 DAG.ReplaceAllUsesOfValueWith(OtherExtend, Extract1);
8689 if (Chain)
8690 DAG.ReplaceAllUsesOfValueWith(OtherExtend.getValue(1), Chain);
8691 SDValue Extract0 =
8692 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f64,
8693 VExtend, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
8694 if (Chain)
8695 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
8696 N->getVTList(), Extract0, Chain);
8697 return Extract0;
8698 }
8699 }
8700 }
8701 }
8702 return SDValue();
8703}
8704
8705SDValue SystemZTargetLowering::combineINT_TO_FP(
8706 SDNode *N, DAGCombinerInfo &DCI) const {
8707 if (DCI.Level != BeforeLegalizeTypes)
8708 return SDValue();
8709 SelectionDAG &DAG = DCI.DAG;
8710 LLVMContext &Ctx = *DAG.getContext();
8711 unsigned Opcode = N->getOpcode();
8712 EVT OutVT = N->getValueType(0);
8713 Type *OutLLVMTy = OutVT.getTypeForEVT(Ctx);
8714 SDValue Op = N->getOperand(0);
8715 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
8716 unsigned InScalarBits = Op->getValueType(0).getScalarSizeInBits();
8717
8718 // Insert an extension before type-legalization to avoid scalarization, e.g.:
8719 // v2f64 = uint_to_fp v2i16
8720 // =>
8721 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
8722 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
8723 OutScalarBits <= 64) {
8724 unsigned NumElts = cast<FixedVectorType>(OutLLVMTy)->getNumElements();
8725 EVT ExtVT = EVT::getVectorVT(
8726 Ctx, EVT::getIntegerVT(Ctx, OutLLVMTy->getScalarSizeInBits()), NumElts);
8727 unsigned ExtOpcode =
8729 SDValue ExtOp = DAG.getNode(ExtOpcode, SDLoc(N), ExtVT, Op);
8730 return DAG.getNode(Opcode, SDLoc(N), OutVT, ExtOp);
8731 }
8732 return SDValue();
8733}
8734
8735SDValue SystemZTargetLowering::combineFCOPYSIGN(
8736 SDNode *N, DAGCombinerInfo &DCI) const {
8737 SelectionDAG &DAG = DCI.DAG;
8738 EVT VT = N->getValueType(0);
8739 SDValue ValOp = N->getOperand(0);
8740 SDValue SignOp = N->getOperand(1);
8741
8742 // Remove the rounding which is not needed.
8743 if (SignOp.getOpcode() == ISD::FP_ROUND) {
8744 SDValue WideOp = SignOp.getOperand(0);
8745 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, ValOp, WideOp);
8746 }
8747
8748 return SDValue();
8749}
8750
8751SDValue SystemZTargetLowering::combineBSWAP(
8752 SDNode *N, DAGCombinerInfo &DCI) const {
8753 SelectionDAG &DAG = DCI.DAG;
8754 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
8755 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
8756 N->getOperand(0).hasOneUse() &&
8757 canLoadStoreByteSwapped(N->getValueType(0))) {
8758 SDValue Load = N->getOperand(0);
8759 LoadSDNode *LD = cast<LoadSDNode>(Load);
8760
8761 // Create the byte-swapping load.
8762 SDValue Ops[] = {
8763 LD->getChain(), // Chain
8764 LD->getBasePtr() // Ptr
8765 };
8766 EVT LoadVT = N->getValueType(0);
8767 if (LoadVT == MVT::i16)
8768 LoadVT = MVT::i32;
8769 SDValue BSLoad =
8770 DAG.getMemIntrinsicNode(SystemZISD::LRV, SDLoc(N),
8771 DAG.getVTList(LoadVT, MVT::Other),
8772 Ops, LD->getMemoryVT(), LD->getMemOperand());
8773
8774 // If this is an i16 load, insert the truncate.
8775 SDValue ResVal = BSLoad;
8776 if (N->getValueType(0) == MVT::i16)
8777 ResVal = DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16, BSLoad);
8778
8779 // First, combine the bswap away. This makes the value produced by the
8780 // load dead.
8781 DCI.CombineTo(N, ResVal);
8782
8783 // Next, combine the load away, we give it a bogus result value but a real
8784 // chain result. The result value is dead because the bswap is dead.
8785 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1));
8786
8787 // Return N so it doesn't get rechecked!
8788 return SDValue(N, 0);
8789 }
8790
8791 // Look through bitcasts that retain the number of vector elements.
8792 SDValue Op = N->getOperand(0);
8793 if (Op.getOpcode() == ISD::BITCAST &&
8794 Op.getValueType().isVector() &&
8795 Op.getOperand(0).getValueType().isVector() &&
8796 Op.getValueType().getVectorNumElements() ==
8797 Op.getOperand(0).getValueType().getVectorNumElements())
8798 Op = Op.getOperand(0);
8799
8800 // Push BSWAP into a vector insertion if at least one side then simplifies.
8801 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
8802 SDValue Vec = Op.getOperand(0);
8803 SDValue Elt = Op.getOperand(1);
8804 SDValue Idx = Op.getOperand(2);
8805
8807 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
8809 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
8810 (canLoadStoreByteSwapped(N->getValueType(0)) &&
8811 ISD::isNON_EXTLoad(Elt.getNode()) && Elt.hasOneUse())) {
8812 EVT VecVT = N->getValueType(0);
8813 EVT EltVT = N->getValueType(0).getVectorElementType();
8814 if (VecVT != Vec.getValueType()) {
8815 Vec = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Vec);
8816 DCI.AddToWorklist(Vec.getNode());
8817 }
8818 if (EltVT != Elt.getValueType()) {
8819 Elt = DAG.getNode(ISD::BITCAST, SDLoc(N), EltVT, Elt);
8820 DCI.AddToWorklist(Elt.getNode());
8821 }
8822 Vec = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Vec);
8823 DCI.AddToWorklist(Vec.getNode());
8824 Elt = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Elt);
8825 DCI.AddToWorklist(Elt.getNode());
8826 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), VecVT,
8827 Vec, Elt, Idx);
8828 }
8829 }
8830
8831 // Push BSWAP into a vector shuffle if at least one side then simplifies.
8832 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Op);
8833 if (SV && Op.hasOneUse()) {
8834 SDValue Op0 = Op.getOperand(0);
8835 SDValue Op1 = Op.getOperand(1);
8836
8838 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
8840 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
8841 EVT VecVT = N->getValueType(0);
8842 if (VecVT != Op0.getValueType()) {
8843 Op0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op0);
8844 DCI.AddToWorklist(Op0.getNode());
8845 }
8846 if (VecVT != Op1.getValueType()) {
8847 Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op1);
8848 DCI.AddToWorklist(Op1.getNode());
8849 }
8850 Op0 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op0);
8851 DCI.AddToWorklist(Op0.getNode());
8852 Op1 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op1);
8853 DCI.AddToWorklist(Op1.getNode());
8854 return DAG.getVectorShuffle(VecVT, SDLoc(N), Op0, Op1, SV->getMask());
8855 }
8856 }
8857
8858 return SDValue();
8859}
8860
8861SDValue SystemZTargetLowering::combineSETCC(
8862 SDNode *N, DAGCombinerInfo &DCI) const {
8863 SelectionDAG &DAG = DCI.DAG;
8864 const ISD::CondCode CC = N->getOperandCondCode(2);
8865 const SDValue LHS = N->getOperand(0);
8866 const SDValue RHS = N->getOperand(1);
8867 bool CmpNull = isNullConstant(RHS);
8868 bool CmpAllOnes = isAllOnesConstant(RHS);
8869 EVT VT = N->getValueType(0);
8870 SDLoc DL(N);
8871
8872 // Match icmp_eq/ne(bitcast(icmp(X,Y)),0/-1) reduction patterns, and
8873 // change the outer compare to a i128 compare. This will normally
8874 // allow the reduction to be recognized in adjustICmp128, and even if
8875 // not, the i128 compare will still generate better code.
8876 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && (CmpNull || CmpAllOnes)) {
8878 if (Src.getOpcode() == ISD::SETCC &&
8879 Src.getValueType().isFixedLengthVector() &&
8880 Src.getValueType().getScalarType() == MVT::i1) {
8881 EVT CmpVT = Src.getOperand(0).getValueType();
8882 if (CmpVT.getSizeInBits() == 128) {
8883 EVT IntVT = CmpVT.changeVectorElementTypeToInteger();
8884 SDValue LHS =
8885 DAG.getBitcast(MVT::i128, DAG.getSExtOrTrunc(Src, DL, IntVT));
8886 SDValue RHS = CmpNull ? DAG.getConstant(0, DL, MVT::i128)
8887 : DAG.getAllOnesConstant(DL, MVT::i128);
8888 return DAG.getNode(ISD::SETCC, DL, VT, LHS, RHS, N->getOperand(2),
8889 N->getFlags());
8890 }
8891 }
8892 }
8893
8894 return SDValue();
8895}
8896
8897static std::pair<SDValue, int> findCCUse(const SDValue &Val,
8898 unsigned Depth = 0) {
8899 // Limit depth of potentially exponential walk.
8900 if (Depth > 5)
8901 return std::make_pair(SDValue(), SystemZ::CCMASK_NONE);
8902
8903 switch (Val.getOpcode()) {
8904 default:
8905 return std::make_pair(SDValue(), SystemZ::CCMASK_NONE);
8906 case SystemZISD::IPM:
8907 if (Val.getOperand(0).getOpcode() == SystemZISD::CLC ||
8908 Val.getOperand(0).getOpcode() == SystemZISD::STRCMP)
8909 return std::make_pair(Val.getOperand(0), SystemZ::CCMASK_ICMP);
8910 return std::make_pair(Val.getOperand(0), SystemZ::CCMASK_ANY);
8911 case SystemZISD::SELECT_CCMASK: {
8912 SDValue Op4CCReg = Val.getOperand(4);
8913 if (Op4CCReg.getOpcode() == SystemZISD::ICMP ||
8914 Op4CCReg.getOpcode() == SystemZISD::TM) {
8915 auto [OpCC, OpCCValid] = findCCUse(Op4CCReg.getOperand(0), Depth + 1);
8916 if (OpCC != SDValue())
8917 return std::make_pair(OpCC, OpCCValid);
8918 }
8919 auto *CCValid = dyn_cast<ConstantSDNode>(Val.getOperand(2));
8920 if (!CCValid)
8921 return std::make_pair(SDValue(), SystemZ::CCMASK_NONE);
8922 int CCValidVal = CCValid->getZExtValue();
8923 return std::make_pair(Op4CCReg, CCValidVal);
8924 }
8925 case ISD::ADD:
8926 case ISD::AND:
8927 case ISD::OR:
8928 case ISD::XOR:
8929 case ISD::SHL:
8930 case ISD::SRA:
8931 case ISD::SRL:
8932 auto [Op0CC, Op0CCValid] = findCCUse(Val.getOperand(0), Depth + 1);
8933 if (Op0CC != SDValue())
8934 return std::make_pair(Op0CC, Op0CCValid);
8935 return findCCUse(Val.getOperand(1), Depth + 1);
8936 }
8937}
8938
8939static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8940 SelectionDAG &DAG);
8941
8943 SelectionDAG &DAG) {
8944 SDLoc DL(Val);
8945 auto Opcode = Val.getOpcode();
8946 switch (Opcode) {
8947 default:
8948 return {};
8949 case ISD::Constant:
8950 return {Val, Val, Val, Val};
8951 case SystemZISD::IPM: {
8952 SDValue IPMOp0 = Val.getOperand(0);
8953 if (IPMOp0 != CC)
8954 return {};
8955 SmallVector<SDValue, 4> ShiftedCCVals;
8956 for (auto CC : {0, 1, 2, 3})
8957 ShiftedCCVals.emplace_back(
8958 DAG.getConstant((CC << SystemZ::IPM_CC), DL, MVT::i32));
8959 return ShiftedCCVals;
8960 }
8961 case SystemZISD::SELECT_CCMASK: {
8962 SDValue TrueVal = Val.getOperand(0), FalseVal = Val.getOperand(1);
8963 auto *CCValid = dyn_cast<ConstantSDNode>(Val.getOperand(2));
8964 auto *CCMask = dyn_cast<ConstantSDNode>(Val.getOperand(3));
8965 if (!CCValid || !CCMask)
8966 return {};
8967
8968 int CCValidVal = CCValid->getZExtValue();
8969 int CCMaskVal = CCMask->getZExtValue();
8970 // Pruning search tree early - Moving CC test and combineCCMask ahead of
8971 // recursive call to simplifyAssumingCCVal.
8972 SDValue Op4CCReg = Val.getOperand(4);
8973 if (Op4CCReg != CC)
8974 combineCCMask(Op4CCReg, CCValidVal, CCMaskVal, DAG);
8975 if (Op4CCReg != CC)
8976 return {};
8977 const auto &&TrueSDVals = simplifyAssumingCCVal(TrueVal, CC, DAG);
8978 const auto &&FalseSDVals = simplifyAssumingCCVal(FalseVal, CC, DAG);
8979 if (TrueSDVals.empty() || FalseSDVals.empty())
8980 return {};
8981 SmallVector<SDValue, 4> MergedSDVals;
8982 for (auto &CCVal : {0, 1, 2, 3})
8983 MergedSDVals.emplace_back(((CCMaskVal & (1 << (3 - CCVal))) != 0)
8984 ? TrueSDVals[CCVal]
8985 : FalseSDVals[CCVal]);
8986 return MergedSDVals;
8987 }
8988 case ISD::ADD:
8989 case ISD::AND:
8990 case ISD::OR:
8991 case ISD::XOR:
8992 case ISD::SRA:
8993 // Avoid introducing CC spills (because ADD/AND/OR/XOR/SRA
8994 // would clobber CC).
8995 if (!Val.hasOneUse())
8996 return {};
8997 [[fallthrough]];
8998 case ISD::SHL:
8999 case ISD::SRL:
9000 SDValue Op0 = Val.getOperand(0), Op1 = Val.getOperand(1);
9001 const auto &&Op0SDVals = simplifyAssumingCCVal(Op0, CC, DAG);
9002 const auto &&Op1SDVals = simplifyAssumingCCVal(Op1, CC, DAG);
9003 if (Op0SDVals.empty() || Op1SDVals.empty())
9004 return {};
9005 SmallVector<SDValue, 4> BinaryOpSDVals;
9006 for (auto CCVal : {0, 1, 2, 3})
9007 BinaryOpSDVals.emplace_back(DAG.getNode(
9008 Opcode, DL, Val.getValueType(), Op0SDVals[CCVal], Op1SDVals[CCVal]));
9009 return BinaryOpSDVals;
9010 }
9011}
9012
9013static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
9014 SelectionDAG &DAG) {
9015 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
9016 // set by the CCReg instruction using the CCValid / CCMask masks,
9017 // If the CCReg instruction is itself a ICMP / TM testing the condition
9018 // code set by some other instruction, see whether we can directly
9019 // use that condition code.
9020 auto *CCNode = CCReg.getNode();
9021 if (!CCNode)
9022 return false;
9023
9024 if (CCNode->getOpcode() == SystemZISD::TM) {
9025 if (CCValid != SystemZ::CCMASK_TM)
9026 return false;
9027 auto emulateTMCCMask = [](const SDValue &Op0Val, const SDValue &Op1Val) {
9028 auto *Op0Node = dyn_cast<ConstantSDNode>(Op0Val.getNode());
9029 auto *Op1Node = dyn_cast<ConstantSDNode>(Op1Val.getNode());
9030 if (!Op0Node || !Op1Node)
9031 return -1;
9032 auto Op0APVal = Op0Node->getAPIntValue();
9033 auto Op1APVal = Op1Node->getAPIntValue();
9034 auto Result = Op0APVal & Op1APVal;
9035 bool AllOnes = Result == Op1APVal;
9036 bool AllZeros = Result == 0;
9037 bool IsLeftMostBitSet = Result[Op1APVal.getActiveBits() - 1] != 0;
9038 return AllZeros ? 0 : AllOnes ? 3 : IsLeftMostBitSet ? 2 : 1;
9039 };
9040 SDValue Op0 = CCNode->getOperand(0);
9041 SDValue Op1 = CCNode->getOperand(1);
9042 auto [Op0CC, Op0CCValid] = findCCUse(Op0);
9043 if (Op0CC == SDValue())
9044 return false;
9045 const auto &&Op0SDVals = simplifyAssumingCCVal(Op0, Op0CC, DAG);
9046 const auto &&Op1SDVals = simplifyAssumingCCVal(Op1, Op0CC, DAG);
9047 if (Op0SDVals.empty() || Op1SDVals.empty())
9048 return false;
9049 int NewCCMask = 0;
9050 for (auto CC : {0, 1, 2, 3}) {
9051 auto CCVal = emulateTMCCMask(Op0SDVals[CC], Op1SDVals[CC]);
9052 if (CCVal < 0)
9053 return false;
9054 NewCCMask <<= 1;
9055 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
9056 }
9057 NewCCMask &= Op0CCValid;
9058 CCReg = Op0CC;
9059 CCMask = NewCCMask;
9060 CCValid = Op0CCValid;
9061 return true;
9062 }
9063 if (CCNode->getOpcode() != SystemZISD::ICMP ||
9064 CCValid != SystemZ::CCMASK_ICMP)
9065 return false;
9066
9067 SDValue CmpOp0 = CCNode->getOperand(0);
9068 SDValue CmpOp1 = CCNode->getOperand(1);
9069 SDValue CmpOp2 = CCNode->getOperand(2);
9070 auto [Op0CC, Op0CCValid] = findCCUse(CmpOp0);
9071 if (Op0CC != SDValue()) {
9072 const auto &&Op0SDVals = simplifyAssumingCCVal(CmpOp0, Op0CC, DAG);
9073 const auto &&Op1SDVals = simplifyAssumingCCVal(CmpOp1, Op0CC, DAG);
9074 if (Op0SDVals.empty() || Op1SDVals.empty())
9075 return false;
9076
9077 auto *CmpType = dyn_cast<ConstantSDNode>(CmpOp2);
9078 auto CmpTypeVal = CmpType->getZExtValue();
9079 const auto compareCCSigned = [&CmpTypeVal](const SDValue &Op0Val,
9080 const SDValue &Op1Val) {
9081 auto *Op0Node = dyn_cast<ConstantSDNode>(Op0Val.getNode());
9082 auto *Op1Node = dyn_cast<ConstantSDNode>(Op1Val.getNode());
9083 if (!Op0Node || !Op1Node)
9084 return -1;
9085 auto Op0APVal = Op0Node->getAPIntValue();
9086 auto Op1APVal = Op1Node->getAPIntValue();
9087 if (CmpTypeVal == SystemZICMP::SignedOnly)
9088 return Op0APVal == Op1APVal ? 0 : Op0APVal.slt(Op1APVal) ? 1 : 2;
9089 return Op0APVal == Op1APVal ? 0 : Op0APVal.ult(Op1APVal) ? 1 : 2;
9090 };
9091 int NewCCMask = 0;
9092 for (auto CC : {0, 1, 2, 3}) {
9093 auto CCVal = compareCCSigned(Op0SDVals[CC], Op1SDVals[CC]);
9094 if (CCVal < 0)
9095 return false;
9096 NewCCMask <<= 1;
9097 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
9098 }
9099 NewCCMask &= Op0CCValid;
9100 CCMask = NewCCMask;
9101 CCReg = Op0CC;
9102 CCValid = Op0CCValid;
9103 return true;
9104 }
9105
9106 return false;
9107}
9108
9109// Merging versus split in multiple branches cost.
9112 const Value *Lhs,
9113 const Value *Rhs,
9114 const Function *) const {
9115 const auto isFlagOutOpCC = [](const Value *V) {
9116 using namespace llvm::PatternMatch;
9117 const Value *RHSVal;
9118 const APInt *RHSC;
9119 if (const auto *I = dyn_cast<Instruction>(V)) {
9120 // PatternMatch.h provides concise tree-based pattern match of llvm IR.
9121 if (match(I->getOperand(0), m_And(m_Value(RHSVal), m_APInt(RHSC))) ||
9122 match(I, m_Cmp(m_Value(RHSVal), m_APInt(RHSC)))) {
9123 if (const auto *CB = dyn_cast<CallBase>(RHSVal)) {
9124 if (CB->isInlineAsm()) {
9125 const InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
9126 return IA && IA->getConstraintString().contains("{@cc}");
9127 }
9128 }
9129 }
9130 }
9131 return false;
9132 };
9133 // Pattern (ICmp %asm) or (ICmp (And %asm)).
9134 // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
9135 // BaseCost can be set >=2. If cost of instruction <= CostThreshold
9136 // conditionals will be merged or else conditionals will be split.
9137 if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
9138 return {3, 0, -1};
9139 // Default.
9140 return {-1, -1, -1};
9141}
9142
9143SDValue SystemZTargetLowering::combineBR_CCMASK(SDNode *N,
9144 DAGCombinerInfo &DCI) const {
9145 SelectionDAG &DAG = DCI.DAG;
9146
9147 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
9148 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));
9149 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));
9150 if (!CCValid || !CCMask)
9151 return SDValue();
9152
9153 int CCValidVal = CCValid->getZExtValue();
9154 int CCMaskVal = CCMask->getZExtValue();
9155 SDValue Chain = N->getOperand(0);
9156 SDValue CCReg = N->getOperand(4);
9157 // If combineCMask was able to merge or simplify ccvalid or ccmask, re-emit
9158 // the modified BR_CCMASK with the new values.
9159 // In order to avoid conditional branches with full or empty cc masks, do not
9160 // do this if ccmask is 0 or equal to ccvalid.
9161 if (combineCCMask(CCReg, CCValidVal, CCMaskVal, DAG) && CCMaskVal != 0 &&
9162 CCMaskVal != CCValidVal)
9163 return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0),
9164 Chain,
9165 DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
9166 DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
9167 N->getOperand(3), CCReg);
9168 return SDValue();
9169}
9170
9171SDValue SystemZTargetLowering::combineSELECT_CCMASK(
9172 SDNode *N, DAGCombinerInfo &DCI) const {
9173 SelectionDAG &DAG = DCI.DAG;
9174
9175 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
9176 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
9177 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
9178 if (!CCValid || !CCMask)
9179 return SDValue();
9180
9181 int CCValidVal = CCValid->getZExtValue();
9182 int CCMaskVal = CCMask->getZExtValue();
9183 SDValue CCReg = N->getOperand(4);
9184
9185 bool IsCombinedCCReg = combineCCMask(CCReg, CCValidVal, CCMaskVal, DAG);
9186
9187 // Populate SDVals vector for each condition code ccval for given Val, which
9188 // can again be another nested select_ccmask with the same CC.
9189 const auto constructCCSDValsFromSELECT = [&CCReg](SDValue &Val) {
9190 if (Val.getOpcode() == SystemZISD::SELECT_CCMASK) {
9192 if (Val.getOperand(4) != CCReg)
9193 return SmallVector<SDValue, 4>{};
9194 SDValue TrueVal = Val.getOperand(0), FalseVal = Val.getOperand(1);
9195 auto *CCMask = dyn_cast<ConstantSDNode>(Val.getOperand(3));
9196 if (!CCMask)
9197 return SmallVector<SDValue, 4>{};
9198
9199 int CCMaskVal = CCMask->getZExtValue();
9200 for (auto &CC : {0, 1, 2, 3})
9201 Res.emplace_back(((CCMaskVal & (1 << (3 - CC))) != 0) ? TrueVal
9202 : FalseVal);
9203 return Res;
9204 }
9205 return SmallVector<SDValue, 4>{Val, Val, Val, Val};
9206 };
9207 // Attempting to optimize TrueVal/FalseVal in outermost select_ccmask either
9208 // with CCReg found by combineCCMask or original CCReg.
9209 SDValue TrueVal = N->getOperand(0);
9210 SDValue FalseVal = N->getOperand(1);
9211 auto &&TrueSDVals = simplifyAssumingCCVal(TrueVal, CCReg, DAG);
9212 auto &&FalseSDVals = simplifyAssumingCCVal(FalseVal, CCReg, DAG);
9213 // TrueSDVals/FalseSDVals might be empty in case of non-constant
9214 // TrueVal/FalseVal for select_ccmask, which can not be optimized further.
9215 if (TrueSDVals.empty())
9216 TrueSDVals = constructCCSDValsFromSELECT(TrueVal);
9217 if (FalseSDVals.empty())
9218 FalseSDVals = constructCCSDValsFromSELECT(FalseVal);
9219 if (!TrueSDVals.empty() && !FalseSDVals.empty()) {
9220 SmallSet<SDValue, 4> MergedSDValsSet;
9221 // Ignoring CC values outside CCValiid.
9222 for (auto CC : {0, 1, 2, 3}) {
9223 if ((CCValidVal & ((1 << (3 - CC)))) != 0)
9224 MergedSDValsSet.insert(((CCMaskVal & (1 << (3 - CC))) != 0)
9225 ? TrueSDVals[CC]
9226 : FalseSDVals[CC]);
9227 }
9228 if (MergedSDValsSet.size() == 1)
9229 return *MergedSDValsSet.begin();
9230 if (MergedSDValsSet.size() == 2) {
9231 auto BeginIt = MergedSDValsSet.begin();
9232 SDValue NewTrueVal = *BeginIt, NewFalseVal = *next(BeginIt);
9233 if (NewTrueVal == FalseVal || NewFalseVal == TrueVal)
9234 std::swap(NewTrueVal, NewFalseVal);
9235 int NewCCMask = 0;
9236 for (auto CC : {0, 1, 2, 3}) {
9237 NewCCMask <<= 1;
9238 NewCCMask |= ((CCMaskVal & (1 << (3 - CC))) != 0)
9239 ? (TrueSDVals[CC] == NewTrueVal)
9240 : (FalseSDVals[CC] == NewTrueVal);
9241 }
9242 CCMaskVal = NewCCMask;
9243 CCMaskVal &= CCValidVal;
9244 TrueVal = NewTrueVal;
9245 FalseVal = NewFalseVal;
9246 IsCombinedCCReg = true;
9247 }
9248 }
9249 // If the condition is trivially false or trivially true after
9250 // combineCCMask, just collapse this SELECT_CCMASK to the indicated value
9251 // (possibly modified by constructCCSDValsFromSELECT).
9252 if (CCMaskVal == 0)
9253 return FalseVal;
9254 if (CCMaskVal == CCValidVal)
9255 return TrueVal;
9256
9257 if (IsCombinedCCReg)
9258 return DAG.getNode(
9259 SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), TrueVal,
9260 FalseVal, DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
9261 DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32), CCReg);
9262
9263 return SDValue();
9264}
9265
9266SDValue SystemZTargetLowering::combineGET_CCMASK(
9267 SDNode *N, DAGCombinerInfo &DCI) const {
9268
9269 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
9270 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));
9271 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));
9272 if (!CCValid || !CCMask)
9273 return SDValue();
9274 int CCValidVal = CCValid->getZExtValue();
9275 int CCMaskVal = CCMask->getZExtValue();
9276
9277 SDValue Select = N->getOperand(0);
9278 if (Select->getOpcode() == ISD::TRUNCATE)
9279 Select = Select->getOperand(0);
9280 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
9281 return SDValue();
9282
9283 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Select->getOperand(2));
9284 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Select->getOperand(3));
9285 if (!SelectCCValid || !SelectCCMask)
9286 return SDValue();
9287 int SelectCCValidVal = SelectCCValid->getZExtValue();
9288 int SelectCCMaskVal = SelectCCMask->getZExtValue();
9289
9290 auto *TrueVal = dyn_cast<ConstantSDNode>(Select->getOperand(0));
9291 auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1));
9292 if (!TrueVal || !FalseVal)
9293 return SDValue();
9294 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
9295 ;
9296 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
9297 SelectCCMaskVal ^= SelectCCValidVal;
9298 else
9299 return SDValue();
9300
9301 if (SelectCCValidVal & ~CCValidVal)
9302 return SDValue();
9303 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
9304 return SDValue();
9305
9306 return Select->getOperand(4);
9307}
9308
9309SDValue SystemZTargetLowering::combineIntDIVREM(
9310 SDNode *N, DAGCombinerInfo &DCI) const {
9311 SelectionDAG &DAG = DCI.DAG;
9312 EVT VT = N->getValueType(0);
9313 // In the case where the divisor is a vector of constants a cheaper
9314 // sequence of instructions can replace the divide. BuildSDIV is called to
9315 // do this during DAG combining, but it only succeeds when it can build a
9316 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
9317 // since it is not Legal but Custom it can only happen before
9318 // legalization. Therefore we must scalarize this early before Combine
9319 // 1. For widened vectors, this is already the result of type legalization.
9320 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
9321 DAG.isConstantIntBuildVectorOrConstantInt(N->getOperand(1)))
9322 return DAG.UnrollVectorOp(N);
9323 return SDValue();
9324}
9325
9326
9327// Transform a right shift of a multiply-and-add into a multiply-and-add-high.
9328// This is closely modeled after the common-code combineShiftToMULH.
9329SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
9330 SDNode *N, DAGCombinerInfo &DCI) const {
9331 SelectionDAG &DAG = DCI.DAG;
9332 SDLoc DL(N);
9333
9334 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
9335 "SRL or SRA node is required here!");
9336
9337 if (!Subtarget.hasVector())
9338 return SDValue();
9339
9340 // Check the shift amount. Proceed with the transformation if the shift
9341 // amount is constant.
9342 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N->getOperand(1));
9343 if (!ShiftAmtSrc)
9344 return SDValue();
9345
9346 // The operation feeding into the shift must be an add.
9347 SDValue ShiftOperand = N->getOperand(0);
9348 if (ShiftOperand.getOpcode() != ISD::ADD)
9349 return SDValue();
9350
9351 // One operand of the add must be a multiply.
9352 SDValue MulOp = ShiftOperand.getOperand(0);
9353 SDValue AddOp = ShiftOperand.getOperand(1);
9354 if (MulOp.getOpcode() != ISD::MUL) {
9355 if (AddOp.getOpcode() != ISD::MUL)
9356 return SDValue();
9357 std::swap(MulOp, AddOp);
9358 }
9359
9360 // All operands must be equivalent extend nodes.
9361 SDValue LeftOp = MulOp.getOperand(0);
9362 SDValue RightOp = MulOp.getOperand(1);
9363
9364 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
9365 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
9366
9367 if (!IsSignExt && !IsZeroExt)
9368 return SDValue();
9369
9370 EVT NarrowVT = LeftOp.getOperand(0).getValueType();
9371 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
9372
9373 SDValue MulhRightOp;
9374 if (ConstantSDNode *Constant = isConstOrConstSplat(RightOp)) {
9375 unsigned ActiveBits = IsSignExt
9376 ? Constant->getAPIntValue().getSignificantBits()
9377 : Constant->getAPIntValue().getActiveBits();
9378 if (ActiveBits > NarrowVTSize)
9379 return SDValue();
9380 MulhRightOp = DAG.getConstant(
9381 Constant->getAPIntValue().trunc(NarrowVT.getScalarSizeInBits()), DL,
9382 NarrowVT);
9383 } else {
9384 if (LeftOp.getOpcode() != RightOp.getOpcode())
9385 return SDValue();
9386 // Check that the two extend nodes are the same type.
9387 if (NarrowVT != RightOp.getOperand(0).getValueType())
9388 return SDValue();
9389 MulhRightOp = RightOp.getOperand(0);
9390 }
9391
9392 SDValue MulhAddOp;
9393 if (ConstantSDNode *Constant = isConstOrConstSplat(AddOp)) {
9394 unsigned ActiveBits = IsSignExt
9395 ? Constant->getAPIntValue().getSignificantBits()
9396 : Constant->getAPIntValue().getActiveBits();
9397 if (ActiveBits > NarrowVTSize)
9398 return SDValue();
9399 MulhAddOp = DAG.getConstant(
9400 Constant->getAPIntValue().trunc(NarrowVT.getScalarSizeInBits()), DL,
9401 NarrowVT);
9402 } else {
9403 if (LeftOp.getOpcode() != AddOp.getOpcode())
9404 return SDValue();
9405 // Check that the two extend nodes are the same type.
9406 if (NarrowVT != AddOp.getOperand(0).getValueType())
9407 return SDValue();
9408 MulhAddOp = AddOp.getOperand(0);
9409 }
9410
9411 EVT WideVT = LeftOp.getValueType();
9412 // Proceed with the transformation if the wide types match.
9413 assert((WideVT == RightOp.getValueType()) &&
9414 "Cannot have a multiply node with two different operand types.");
9415 assert((WideVT == AddOp.getValueType()) &&
9416 "Cannot have an add node with two different operand types.");
9417
9418 // Proceed with the transformation if the wide type is twice as large
9419 // as the narrow type.
9420 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
9421 return SDValue();
9422
9423 // Check the shift amount with the narrow type size.
9424 // Proceed with the transformation if the shift amount is the width
9425 // of the narrow type.
9426 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
9427 if (ShiftAmt != NarrowVTSize)
9428 return SDValue();
9429
9430 // Proceed if we support the multiply-and-add-high operation.
9431 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9432 NarrowVT == MVT::v4i32 ||
9433 (Subtarget.hasVectorEnhancements3() &&
9434 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9435 return SDValue();
9436
9437 // Emit the VMAH (signed) or VMALH (unsigned) operation.
9438 SDValue Result = DAG.getNode(IsSignExt ? SystemZISD::VMAH : SystemZISD::VMALH,
9439 DL, NarrowVT, LeftOp.getOperand(0),
9440 MulhRightOp, MulhAddOp);
9441 bool IsSigned = N->getOpcode() == ISD::SRA;
9442 return DAG.getExtOrTrunc(IsSigned, Result, DL, WideVT);
9443}
9444
9445// Op is an operand of a multiplication. Check whether this can be folded
9446// into an even/odd widening operation; if so, return the opcode to be used
9447// and update Op to the appropriate sub-operand. Note that the caller must
9448// verify that *both* operands of the multiplication support the operation.
9451 SDValue &Op) {
9452 EVT VT = Op.getValueType();
9453
9454 // Check for (sign/zero_extend_vector_inreg (vector_shuffle)) corresponding
9455 // to selecting the even or odd vector elements.
9456 if (VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
9457 (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
9458 Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)) {
9459 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
9460 unsigned NumElts = VT.getVectorNumElements();
9461 Op = Op.getOperand(0);
9462 if (Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9463 Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
9465 ArrayRef<int> ShuffleMask = SVN->getMask();
9466 bool CanUseEven = true, CanUseOdd = true;
9467 for (unsigned Elt = 0; Elt < NumElts; Elt++) {
9468 if (ShuffleMask[Elt] == -1)
9469 continue;
9470 if (unsigned(ShuffleMask[Elt]) != 2 * Elt)
9471 CanUseEven = false;
9472 if (unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9473 CanUseOdd = false;
9474 }
9475 Op = Op.getOperand(0);
9476 if (CanUseEven)
9477 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9478 if (CanUseOdd)
9479 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9480 }
9481 }
9482
9483 // For z17, we can also support the v2i64->i128 case, which looks like
9484 // (sign/zero_extend (extract_vector_elt X 0/1))
9485 if (VT == MVT::i128 && Subtarget.hasVectorEnhancements3() &&
9486 (Op.getOpcode() == ISD::SIGN_EXTEND ||
9487 Op.getOpcode() == ISD::ZERO_EXTEND)) {
9488 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND;
9489 Op = Op.getOperand(0);
9490 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
9491 Op.getOperand(0).getValueType() == MVT::v2i64 &&
9492 Op.getOperand(1).getOpcode() == ISD::Constant) {
9493 unsigned Elem = Op.getConstantOperandVal(1);
9494 Op = Op.getOperand(0);
9495 if (Elem == 0)
9496 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9497 if (Elem == 1)
9498 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9499 }
9500 }
9501
9502 return 0;
9503}
9504
9505SDValue SystemZTargetLowering::combineMUL(
9506 SDNode *N, DAGCombinerInfo &DCI) const {
9507 SelectionDAG &DAG = DCI.DAG;
9508
9509 // Detect even/odd widening multiplication.
9510 SDValue Op0 = N->getOperand(0);
9511 SDValue Op1 = N->getOperand(1);
9512 unsigned OpcodeCand0 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op0);
9513 unsigned OpcodeCand1 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op1);
9514 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9515 return DAG.getNode(OpcodeCand0, SDLoc(N), N->getValueType(0), Op0, Op1);
9516
9517 return SDValue();
9518}
9519
9520SDValue SystemZTargetLowering::combineINTRINSIC(
9521 SDNode *N, DAGCombinerInfo &DCI) const {
9522 SelectionDAG &DAG = DCI.DAG;
9523
9524 unsigned Id = N->getConstantOperandVal(1);
9525 switch (Id) {
9526 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
9527 // or larger is simply a vector load.
9528 case Intrinsic::s390_vll:
9529 case Intrinsic::s390_vlrl:
9530 if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(2)))
9531 if (C->getZExtValue() >= 15)
9532 return DAG.getLoad(N->getValueType(0), SDLoc(N), N->getOperand(0),
9533 N->getOperand(3), MachinePointerInfo());
9534 break;
9535 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
9536 case Intrinsic::s390_vstl:
9537 case Intrinsic::s390_vstrl:
9538 if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(3)))
9539 if (C->getZExtValue() >= 15)
9540 return DAG.getStore(N->getOperand(0), SDLoc(N), N->getOperand(2),
9541 N->getOperand(4), MachinePointerInfo());
9542 break;
9543 }
9544
9545 return SDValue();
9546}
9547
9548SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
9549 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9550 return N->getOperand(0);
9551 return N;
9552}
9553
9555 DAGCombinerInfo &DCI) const {
9556 switch(N->getOpcode()) {
9557 default: break;
9558 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
9559 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
9560 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
9561 case SystemZISD::MERGE_HIGH:
9562 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
9563 case ISD::LOAD: return combineLOAD(N, DCI);
9564 case ISD::STORE: return combineSTORE(N, DCI);
9565 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
9566 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
9567 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
9569 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
9571 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
9572 case ISD::SINT_TO_FP:
9573 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
9574 case ISD::FCOPYSIGN: return combineFCOPYSIGN(N, DCI);
9575 case ISD::BSWAP: return combineBSWAP(N, DCI);
9576 case ISD::SETCC: return combineSETCC(N, DCI);
9577 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
9578 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
9579 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
9580 case ISD::SRL:
9581 case ISD::SRA: return combineShiftToMulAddHigh(N, DCI);
9582 case ISD::MUL: return combineMUL(N, DCI);
9583 case ISD::SDIV:
9584 case ISD::UDIV:
9585 case ISD::SREM:
9586 case ISD::UREM: return combineIntDIVREM(N, DCI);
9588 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
9589 }
9590
9591 return SDValue();
9592}
9593
9594// Return the demanded elements for the OpNo source operand of Op. DemandedElts
9595// are for Op.
9596static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
9597 unsigned OpNo) {
9598 EVT VT = Op.getValueType();
9599 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
9600 APInt SrcDemE;
9601 unsigned Opcode = Op.getOpcode();
9602 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9603 unsigned Id = Op.getConstantOperandVal(0);
9604 switch (Id) {
9605 case Intrinsic::s390_vpksh: // PACKS
9606 case Intrinsic::s390_vpksf:
9607 case Intrinsic::s390_vpksg:
9608 case Intrinsic::s390_vpkshs: // PACKS_CC
9609 case Intrinsic::s390_vpksfs:
9610 case Intrinsic::s390_vpksgs:
9611 case Intrinsic::s390_vpklsh: // PACKLS
9612 case Intrinsic::s390_vpklsf:
9613 case Intrinsic::s390_vpklsg:
9614 case Intrinsic::s390_vpklshs: // PACKLS_CC
9615 case Intrinsic::s390_vpklsfs:
9616 case Intrinsic::s390_vpklsgs:
9617 // VECTOR PACK truncates the elements of two source vectors into one.
9618 SrcDemE = DemandedElts;
9619 if (OpNo == 2)
9620 SrcDemE.lshrInPlace(NumElts / 2);
9621 SrcDemE = SrcDemE.trunc(NumElts / 2);
9622 break;
9623 // VECTOR UNPACK extends half the elements of the source vector.
9624 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9625 case Intrinsic::s390_vuphh:
9626 case Intrinsic::s390_vuphf:
9627 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9628 case Intrinsic::s390_vuplhh:
9629 case Intrinsic::s390_vuplhf:
9630 SrcDemE = APInt(NumElts * 2, 0);
9631 SrcDemE.insertBits(DemandedElts, 0);
9632 break;
9633 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9634 case Intrinsic::s390_vuplhw:
9635 case Intrinsic::s390_vuplf:
9636 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9637 case Intrinsic::s390_vupllh:
9638 case Intrinsic::s390_vupllf:
9639 SrcDemE = APInt(NumElts * 2, 0);
9640 SrcDemE.insertBits(DemandedElts, NumElts);
9641 break;
9642 case Intrinsic::s390_vpdi: {
9643 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
9644 SrcDemE = APInt(NumElts, 0);
9645 if (!DemandedElts[OpNo - 1])
9646 break;
9647 unsigned Mask = Op.getConstantOperandVal(3);
9648 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9649 // Demand input element 0 or 1, given by the mask bit value.
9650 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
9651 break;
9652 }
9653 case Intrinsic::s390_vsldb: {
9654 // VECTOR SHIFT LEFT DOUBLE BY BYTE
9655 assert(VT == MVT::v16i8 && "Unexpected type.");
9656 unsigned FirstIdx = Op.getConstantOperandVal(3);
9657 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
9658 unsigned NumSrc0Els = 16 - FirstIdx;
9659 SrcDemE = APInt(NumElts, 0);
9660 if (OpNo == 1) {
9661 APInt DemEls = DemandedElts.trunc(NumSrc0Els);
9662 SrcDemE.insertBits(DemEls, FirstIdx);
9663 } else {
9664 APInt DemEls = DemandedElts.lshr(NumSrc0Els);
9665 SrcDemE.insertBits(DemEls, 0);
9666 }
9667 break;
9668 }
9669 case Intrinsic::s390_vperm:
9670 SrcDemE = APInt::getAllOnes(NumElts);
9671 break;
9672 default:
9673 llvm_unreachable("Unhandled intrinsic.");
9674 break;
9675 }
9676 } else {
9677 switch (Opcode) {
9678 case SystemZISD::JOIN_DWORDS:
9679 // Scalar operand.
9680 SrcDemE = APInt(1, 1);
9681 break;
9682 case SystemZISD::SELECT_CCMASK:
9683 SrcDemE = DemandedElts;
9684 break;
9685 default:
9686 llvm_unreachable("Unhandled opcode.");
9687 break;
9688 }
9689 }
9690 return SrcDemE;
9691}
9692
9694 const APInt &DemandedElts,
9695 const SelectionDAG &DAG, unsigned Depth,
9696 unsigned OpNo) {
9697 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9698 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo + 1);
9699 KnownBits LHSKnown =
9700 DAG.computeKnownBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
9701 KnownBits RHSKnown =
9702 DAG.computeKnownBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
9703 Known = LHSKnown.intersectWith(RHSKnown);
9704}
9705
9706void
9709 const APInt &DemandedElts,
9710 const SelectionDAG &DAG,
9711 unsigned Depth) const {
9712 Known.resetAll();
9713
9714 // Intrinsic CC result is returned in the two low bits.
9715 unsigned Tmp0, Tmp1; // not used
9716 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, Tmp0, Tmp1)) {
9717 Known.Zero.setBitsFrom(2);
9718 return;
9719 }
9720 [[maybe_unused]] EVT VT = Op.getValueType();
9721 if (Op.getResNo() != 0)
9722 return;
9723 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
9724 "KnownBits does not match VT in bitwidth");
9725 assert ((!VT.isVector() ||
9726 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
9727 "DemandedElts does not match VT number of elements");
9728 unsigned BitWidth = Known.getBitWidth();
9729 unsigned Opcode = Op.getOpcode();
9730 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9731 bool IsLogical = false;
9732 unsigned Id = Op.getConstantOperandVal(0);
9733 switch (Id) {
9734 case Intrinsic::s390_vpksh: // PACKS
9735 case Intrinsic::s390_vpksf:
9736 case Intrinsic::s390_vpksg:
9737 case Intrinsic::s390_vpkshs: // PACKS_CC
9738 case Intrinsic::s390_vpksfs:
9739 case Intrinsic::s390_vpksgs:
9740 case Intrinsic::s390_vpklsh: // PACKLS
9741 case Intrinsic::s390_vpklsf:
9742 case Intrinsic::s390_vpklsg:
9743 case Intrinsic::s390_vpklshs: // PACKLS_CC
9744 case Intrinsic::s390_vpklsfs:
9745 case Intrinsic::s390_vpklsgs:
9746 case Intrinsic::s390_vpdi:
9747 case Intrinsic::s390_vsldb:
9748 case Intrinsic::s390_vperm:
9749 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 1);
9750 break;
9751 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9752 case Intrinsic::s390_vuplhh:
9753 case Intrinsic::s390_vuplhf:
9754 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9755 case Intrinsic::s390_vupllh:
9756 case Intrinsic::s390_vupllf:
9757 IsLogical = true;
9758 [[fallthrough]];
9759 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9760 case Intrinsic::s390_vuphh:
9761 case Intrinsic::s390_vuphf:
9762 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9763 case Intrinsic::s390_vuplhw:
9764 case Intrinsic::s390_vuplf: {
9765 SDValue SrcOp = Op.getOperand(1);
9766 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, 0);
9767 Known = DAG.computeKnownBits(SrcOp, SrcDemE, Depth + 1);
9768 if (IsLogical) {
9769 Known = Known.zext(BitWidth);
9770 } else
9771 Known = Known.sext(BitWidth);
9772 break;
9773 }
9774 default:
9775 break;
9776 }
9777 } else {
9778 switch (Opcode) {
9779 case SystemZISD::JOIN_DWORDS:
9780 case SystemZISD::SELECT_CCMASK:
9781 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 0);
9782 break;
9783 case SystemZISD::REPLICATE: {
9784 SDValue SrcOp = Op.getOperand(0);
9785 Known = DAG.computeKnownBits(SrcOp, Depth + 1);
9786 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(SrcOp))
9787 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
9788 break;
9789 }
9790 default:
9791 break;
9792 }
9793 }
9794
9795 // Known has the width of the source operand(s). Adjust if needed to match
9796 // the passed bitwidth.
9797 if (Known.getBitWidth() != BitWidth)
9798 Known = Known.anyextOrTrunc(BitWidth);
9799}
9800
9801static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
9802 const SelectionDAG &DAG, unsigned Depth,
9803 unsigned OpNo) {
9804 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9805 unsigned LHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
9806 if (LHS == 1) return 1; // Early out.
9807 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo + 1);
9808 unsigned RHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
9809 if (RHS == 1) return 1; // Early out.
9810 unsigned Common = std::min(LHS, RHS);
9811 unsigned SrcBitWidth = Op.getOperand(OpNo).getScalarValueSizeInBits();
9812 EVT VT = Op.getValueType();
9813 unsigned VTBits = VT.getScalarSizeInBits();
9814 if (SrcBitWidth > VTBits) { // PACK
9815 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9816 if (Common > SrcExtraBits)
9817 return (Common - SrcExtraBits);
9818 return 1;
9819 }
9820 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
9821 return Common;
9822}
9823
9824unsigned
9826 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9827 unsigned Depth) const {
9828 if (Op.getResNo() != 0)
9829 return 1;
9830 unsigned Opcode = Op.getOpcode();
9831 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9832 unsigned Id = Op.getConstantOperandVal(0);
9833 switch (Id) {
9834 case Intrinsic::s390_vpksh: // PACKS
9835 case Intrinsic::s390_vpksf:
9836 case Intrinsic::s390_vpksg:
9837 case Intrinsic::s390_vpkshs: // PACKS_CC
9838 case Intrinsic::s390_vpksfs:
9839 case Intrinsic::s390_vpksgs:
9840 case Intrinsic::s390_vpklsh: // PACKLS
9841 case Intrinsic::s390_vpklsf:
9842 case Intrinsic::s390_vpklsg:
9843 case Intrinsic::s390_vpklshs: // PACKLS_CC
9844 case Intrinsic::s390_vpklsfs:
9845 case Intrinsic::s390_vpklsgs:
9846 case Intrinsic::s390_vpdi:
9847 case Intrinsic::s390_vsldb:
9848 case Intrinsic::s390_vperm:
9849 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 1);
9850 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9851 case Intrinsic::s390_vuphh:
9852 case Intrinsic::s390_vuphf:
9853 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9854 case Intrinsic::s390_vuplhw:
9855 case Intrinsic::s390_vuplf: {
9856 SDValue PackedOp = Op.getOperand(1);
9857 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, 1);
9858 unsigned Tmp = DAG.ComputeNumSignBits(PackedOp, SrcDemE, Depth + 1);
9859 EVT VT = Op.getValueType();
9860 unsigned VTBits = VT.getScalarSizeInBits();
9861 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
9862 return Tmp;
9863 }
9864 default:
9865 break;
9866 }
9867 } else {
9868 switch (Opcode) {
9869 case SystemZISD::SELECT_CCMASK:
9870 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 0);
9871 default:
9872 break;
9873 }
9874 }
9875
9876 return 1;
9877}
9878
9880 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9881 UndefPoisonKind Kind, unsigned Depth) const {
9882 switch (Op->getOpcode()) {
9883 case SystemZISD::PCREL_WRAPPER:
9884 case SystemZISD::PCREL_OFFSET:
9885 return true;
9886 }
9887 return false;
9888}
9889
9890unsigned
9892 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9893 unsigned StackAlign = TFI->getStackAlignment();
9894 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
9895 "Unexpected stack alignment");
9896 // The default stack probe size is 4096 if the function has no
9897 // stack-probe-size attribute.
9898 unsigned StackProbeSize =
9899 MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size", 4096);
9900 // Round down to the stack alignment.
9901 StackProbeSize &= ~(StackAlign - 1);
9902 return StackProbeSize ? StackProbeSize : StackAlign;
9903}
9904
9905//===----------------------------------------------------------------------===//
9906// Custom insertion
9907//===----------------------------------------------------------------------===//
9908
9909// Force base value Base into a register before MI. Return the register.
9911 const SystemZInstrInfo *TII) {
9912 MachineBasicBlock *MBB = MI.getParent();
9913 MachineFunction &MF = *MBB->getParent();
9915
9916 if (Base.isReg()) {
9917 // Copy Base into a new virtual register to help register coalescing in
9918 // cases with multiple uses.
9919 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9920 BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::COPY), Reg)
9921 .add(Base);
9922 return Reg;
9923 }
9924
9925 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9926 BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LA), Reg)
9927 .add(Base)
9928 .addImm(0)
9929 .addReg(0);
9930 return Reg;
9931}
9932
9933// The CC operand of MI might be missing a kill marker because there
9934// were multiple uses of CC, and ISel didn't know which to mark.
9935// Figure out whether MI should have had a kill marker.
9937 // Scan forward through BB for a use/def of CC.
9939 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
9940 const MachineInstr &MI = *miI;
9941 if (MI.readsRegister(SystemZ::CC, /*TRI=*/nullptr))
9942 return false;
9943 if (MI.definesRegister(SystemZ::CC, /*TRI=*/nullptr))
9944 break; // Should have kill-flag - update below.
9945 }
9946
9947 // If we hit the end of the block, check whether CC is live into a
9948 // successor.
9949 if (miI == MBB->end()) {
9950 for (const MachineBasicBlock *Succ : MBB->successors())
9951 if (Succ->isLiveIn(SystemZ::CC))
9952 return false;
9953 }
9954
9955 return true;
9956}
9957
9958// Return true if it is OK for this Select pseudo-opcode to be cascaded
9959// together with other Select pseudo-opcodes into a single basic-block with
9960// a conditional jump around it.
9962 switch (MI.getOpcode()) {
9963 case SystemZ::Select32:
9964 case SystemZ::Select64:
9965 case SystemZ::Select128:
9966 case SystemZ::SelectF32:
9967 case SystemZ::SelectF64:
9968 case SystemZ::SelectF128:
9969 case SystemZ::SelectVR32:
9970 case SystemZ::SelectVR64:
9971 case SystemZ::SelectVR128:
9972 return true;
9973
9974 default:
9975 return false;
9976 }
9977}
9978
9979// Helper function, which inserts PHI functions into SinkMBB:
9980// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
9981// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
9983 MachineBasicBlock *TrueMBB,
9984 MachineBasicBlock *FalseMBB,
9985 MachineBasicBlock *SinkMBB) {
9986 MachineFunction *MF = TrueMBB->getParent();
9988
9989 MachineInstr *FirstMI = Selects.front();
9990 unsigned CCValid = FirstMI->getOperand(3).getImm();
9991 unsigned CCMask = FirstMI->getOperand(4).getImm();
9992
9993 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
9994
9995 // As we are creating the PHIs, we have to be careful if there is more than
9996 // one. Later Selects may reference the results of earlier Selects, but later
9997 // PHIs have to reference the individual true/false inputs from earlier PHIs.
9998 // That also means that PHI construction must work forward from earlier to
9999 // later, and that the code must maintain a mapping from earlier PHI's
10000 // destination registers, and the registers that went into the PHI.
10002
10003 for (auto *MI : Selects) {
10004 Register DestReg = MI->getOperand(0).getReg();
10005 Register TrueReg = MI->getOperand(1).getReg();
10006 Register FalseReg = MI->getOperand(2).getReg();
10007
10008 // If this Select we are generating is the opposite condition from
10009 // the jump we generated, then we have to swap the operands for the
10010 // PHI that is going to be generated.
10011 if (MI->getOperand(4).getImm() == (CCValid ^ CCMask))
10012 std::swap(TrueReg, FalseReg);
10013
10014 if (auto It = RegRewriteTable.find(TrueReg); It != RegRewriteTable.end())
10015 TrueReg = It->second.first;
10016
10017 if (auto It = RegRewriteTable.find(FalseReg); It != RegRewriteTable.end())
10018 FalseReg = It->second.second;
10019
10020 DebugLoc DL = MI->getDebugLoc();
10021 BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg)
10022 .addReg(TrueReg).addMBB(TrueMBB)
10023 .addReg(FalseReg).addMBB(FalseMBB);
10024
10025 // Add this PHI to the rewrite table.
10026 RegRewriteTable[DestReg] = std::make_pair(TrueReg, FalseReg);
10027 }
10028
10029 MF->getProperties().resetNoPHIs();
10030}
10031
10033SystemZTargetLowering::emitAdjCallStack(MachineInstr &MI,
10034 MachineBasicBlock *BB) const {
10035 MachineFunction &MF = *BB->getParent();
10036 MachineFrameInfo &MFI = MF.getFrameInfo();
10037 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
10038 assert(TFL->hasReservedCallFrame(MF) &&
10039 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
10040 (void)TFL;
10041 // Get the MaxCallFrameSize value and erase MI since it serves no further
10042 // purpose as the call frame is statically reserved in the prolog. Set
10043 // AdjustsStack as MI is *not* mapped as a frame instruction.
10044 uint32_t NumBytes = MI.getOperand(0).getImm();
10045 if (NumBytes > MFI.getMaxCallFrameSize())
10046 MFI.setMaxCallFrameSize(NumBytes);
10047 MFI.setAdjustsStack(true);
10048
10049 MI.eraseFromParent();
10050 return BB;
10051}
10052
10053// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
10055SystemZTargetLowering::emitSelect(MachineInstr &MI,
10056 MachineBasicBlock *MBB) const {
10057 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
10058 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10059
10060 unsigned CCValid = MI.getOperand(3).getImm();
10061 unsigned CCMask = MI.getOperand(4).getImm();
10062
10063 // If we have a sequence of Select* pseudo instructions using the
10064 // same condition code value, we want to expand all of them into
10065 // a single pair of basic blocks using the same condition.
10066 SmallVector<MachineInstr*, 8> Selects;
10067 SmallVector<MachineInstr*, 8> DbgValues;
10068 Selects.push_back(&MI);
10069 unsigned Count = 0;
10070 for (MachineInstr &NextMI : llvm::make_range(
10071 std::next(MachineBasicBlock::iterator(MI)), MBB->end())) {
10072 if (isSelectPseudo(NextMI)) {
10073 assert(NextMI.getOperand(3).getImm() == CCValid &&
10074 "Bad CCValid operands since CC was not redefined.");
10075 if (NextMI.getOperand(4).getImm() == CCMask ||
10076 NextMI.getOperand(4).getImm() == (CCValid ^ CCMask)) {
10077 Selects.push_back(&NextMI);
10078 continue;
10079 }
10080 break;
10081 }
10082 if (NextMI.definesRegister(SystemZ::CC, /*TRI=*/nullptr) ||
10083 NextMI.usesCustomInsertionHook())
10084 break;
10085 bool User = false;
10086 for (auto *SelMI : Selects)
10087 if (NextMI.readsVirtualRegister(SelMI->getOperand(0).getReg())) {
10088 User = true;
10089 break;
10090 }
10091 if (NextMI.isDebugInstr()) {
10092 if (User) {
10093 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
10094 DbgValues.push_back(&NextMI);
10095 }
10096 } else if (User || ++Count > 20)
10097 break;
10098 }
10099
10100 MachineInstr *LastMI = Selects.back();
10101 bool CCKilled = (LastMI->killsRegister(SystemZ::CC, /*TRI=*/nullptr) ||
10102 checkCCKill(*LastMI, MBB));
10103 MachineBasicBlock *StartMBB = MBB;
10104 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(LastMI, MBB);
10105 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(StartMBB);
10106
10107 // Unless CC was killed in the last Select instruction, mark it as
10108 // live-in to both FalseMBB and JoinMBB.
10109 if (!CCKilled) {
10110 FalseMBB->addLiveIn(SystemZ::CC);
10111 JoinMBB->addLiveIn(SystemZ::CC);
10112 }
10113
10114 // StartMBB:
10115 // BRC CCMask, JoinMBB
10116 // # fallthrough to FalseMBB
10117 MBB = StartMBB;
10118 BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))
10119 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
10120 MBB->addSuccessor(JoinMBB);
10121 MBB->addSuccessor(FalseMBB);
10122
10123 // FalseMBB:
10124 // # fallthrough to JoinMBB
10125 MBB = FalseMBB;
10126 MBB->addSuccessor(JoinMBB);
10127
10128 // JoinMBB:
10129 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
10130 // ...
10131 MBB = JoinMBB;
10132 createPHIsForSelects(Selects, StartMBB, FalseMBB, MBB);
10133 for (auto *SelMI : Selects)
10134 SelMI->eraseFromParent();
10135
10137 for (auto *DbgMI : DbgValues)
10138 MBB->splice(InsertPos, StartMBB, DbgMI);
10139
10140 return JoinMBB;
10141}
10142
10143// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
10144// StoreOpcode is the store to use and Invert says whether the store should
10145// happen when the condition is false rather than true. If a STORE ON
10146// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
10147MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
10149 unsigned StoreOpcode,
10150 unsigned STOCOpcode,
10151 bool Invert) const {
10152 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10153
10154 Register SrcReg = MI.getOperand(0).getReg();
10155 MachineOperand Base = MI.getOperand(1);
10156 int64_t Disp = MI.getOperand(2).getImm();
10157 Register IndexReg = MI.getOperand(3).getReg();
10158 unsigned CCValid = MI.getOperand(4).getImm();
10159 unsigned CCMask = MI.getOperand(5).getImm();
10160 DebugLoc DL = MI.getDebugLoc();
10161
10162 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp);
10163
10164 // ISel pattern matching also adds a load memory operand of the same
10165 // address, so take special care to find the storing memory operand.
10166 MachineMemOperand *MMO = nullptr;
10167 for (auto *I : MI.memoperands())
10168 if (I->isStore()) {
10169 MMO = I;
10170 break;
10171 }
10172
10173 // Use STOCOpcode if possible. We could use different store patterns in
10174 // order to avoid matching the index register, but the performance trade-offs
10175 // might be more complicated in that case.
10176 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
10177 if (Invert)
10178 CCMask ^= CCValid;
10179
10180 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode))
10181 .addReg(SrcReg)
10182 .add(Base)
10183 .addImm(Disp)
10184 .addImm(CCValid)
10185 .addImm(CCMask)
10186 .addMemOperand(MMO);
10187
10188 MI.eraseFromParent();
10189 return MBB;
10190 }
10191
10192 // Get the condition needed to branch around the store.
10193 if (!Invert)
10194 CCMask ^= CCValid;
10195
10196 MachineBasicBlock *StartMBB = MBB;
10197 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
10198 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(StartMBB);
10199
10200 // Unless CC was killed in the CondStore instruction, mark it as
10201 // live-in to both FalseMBB and JoinMBB.
10202 if (!MI.killsRegister(SystemZ::CC, /*TRI=*/nullptr) &&
10203 !checkCCKill(MI, JoinMBB)) {
10204 FalseMBB->addLiveIn(SystemZ::CC);
10205 JoinMBB->addLiveIn(SystemZ::CC);
10206 }
10207
10208 // StartMBB:
10209 // BRC CCMask, JoinMBB
10210 // # fallthrough to FalseMBB
10211 MBB = StartMBB;
10212 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10213 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
10214 MBB->addSuccessor(JoinMBB);
10215 MBB->addSuccessor(FalseMBB);
10216
10217 // FalseMBB:
10218 // store %SrcReg, %Disp(%Index,%Base)
10219 // # fallthrough to JoinMBB
10220 MBB = FalseMBB;
10221 BuildMI(MBB, DL, TII->get(StoreOpcode))
10222 .addReg(SrcReg)
10223 .add(Base)
10224 .addImm(Disp)
10225 .addReg(IndexReg)
10226 .addMemOperand(MMO);
10227 MBB->addSuccessor(JoinMBB);
10228
10229 MI.eraseFromParent();
10230 return JoinMBB;
10231}
10232
10233// Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
10235SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
10237 bool Unsigned) const {
10238 MachineFunction &MF = *MBB->getParent();
10239 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10240 MachineRegisterInfo &MRI = MF.getRegInfo();
10241
10242 // Synthetic instruction to compare 128-bit values.
10243 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
10244 Register Op0 = MI.getOperand(0).getReg();
10245 Register Op1 = MI.getOperand(1).getReg();
10246
10247 MachineBasicBlock *StartMBB = MBB;
10248 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
10249 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(StartMBB);
10250
10251 // StartMBB:
10252 //
10253 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
10254 // Swap the inputs to get:
10255 // CC 1 if high(Op0) > high(Op1)
10256 // CC 2 if high(Op0) < high(Op1)
10257 // CC 0 if high(Op0) == high(Op1)
10258 //
10259 // If CC != 0, we'd done, so jump over the next instruction.
10260 //
10261 // VEC[L]G Op1, Op0
10262 // JNE JoinMBB
10263 // # fallthrough to HiEqMBB
10264 MBB = StartMBB;
10265 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
10266 BuildMI(MBB, MI.getDebugLoc(), TII->get(HiOpcode))
10267 .addReg(Op1).addReg(Op0);
10268 BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))
10270 MBB->addSuccessor(JoinMBB);
10271 MBB->addSuccessor(HiEqMBB);
10272
10273 // HiEqMBB:
10274 //
10275 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
10276 // Since we already know the high parts are equal, the CC
10277 // result will only depend on the low parts:
10278 // CC 1 if low(Op0) > low(Op1)
10279 // CC 3 if low(Op0) <= low(Op1)
10280 //
10281 // VCHLGS Tmp, Op0, Op1
10282 // # fallthrough to JoinMBB
10283 MBB = HiEqMBB;
10284 Register Temp = MRI.createVirtualRegister(&SystemZ::VR128BitRegClass);
10285 BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::VCHLGS), Temp)
10286 .addReg(Op0).addReg(Op1);
10287 MBB->addSuccessor(JoinMBB);
10288
10289 // Mark CC as live-in to JoinMBB.
10290 JoinMBB->addLiveIn(SystemZ::CC);
10291
10292 MI.eraseFromParent();
10293 return JoinMBB;
10294}
10295
10296// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
10297// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
10298// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
10299// whether the field should be inverted after performing BinOpcode (e.g. for
10300// NAND).
10301MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
10302 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
10303 bool Invert) const {
10304 MachineFunction &MF = *MBB->getParent();
10305 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10306 MachineRegisterInfo &MRI = MF.getRegInfo();
10307
10308 // Extract the operands. Base can be a register or a frame index.
10309 // Src2 can be a register or immediate.
10310 Register Dest = MI.getOperand(0).getReg();
10311 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
10312 int64_t Disp = MI.getOperand(2).getImm();
10313 MachineOperand Src2 = earlyUseOperand(MI.getOperand(3));
10314 Register BitShift = MI.getOperand(4).getReg();
10315 Register NegBitShift = MI.getOperand(5).getReg();
10316 unsigned BitSize = MI.getOperand(6).getImm();
10317 DebugLoc DL = MI.getDebugLoc();
10318
10319 // Get the right opcodes for the displacement.
10320 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
10321 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
10322 assert(LOpcode && CSOpcode && "Displacement out of range");
10323
10324 // Create virtual registers for temporary results.
10325 Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10326 Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10327 Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10328 Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10329 Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10330
10331 // Insert a basic block for the main loop.
10332 MachineBasicBlock *StartMBB = MBB;
10333 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10334 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
10335
10336 // StartMBB:
10337 // ...
10338 // %OrigVal = L Disp(%Base)
10339 // # fall through to LoopMBB
10340 MBB = StartMBB;
10341 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
10342 MBB->addSuccessor(LoopMBB);
10343
10344 // LoopMBB:
10345 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
10346 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10347 // %RotatedNewVal = OP %RotatedOldVal, %Src2
10348 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10349 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10350 // JNE LoopMBB
10351 // # fall through to DoneMBB
10352 MBB = LoopMBB;
10353 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
10354 .addReg(OrigVal).addMBB(StartMBB)
10355 .addReg(Dest).addMBB(LoopMBB);
10356 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
10357 .addReg(OldVal).addReg(BitShift).addImm(0);
10358 if (Invert) {
10359 // Perform the operation normally and then invert every bit of the field.
10360 Register Tmp = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10361 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp)
10362 .addReg(RotatedOldVal)
10363 .add(Src2)
10364 .setOperandDead(3);
10365 // XILF with the upper BitSize bits set.
10366 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal)
10367 .addReg(Tmp)
10368 .addImm(-1U << (32 - BitSize))
10369 .setOperandDead(3);
10370 } else if (BinOpcode)
10371 // A simply binary operation.
10372 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal)
10373 .addReg(RotatedOldVal)
10374 .add(Src2)
10375 .setOperandDead(3);
10376 else
10377 // Use RISBG to rotate Src2 into position and use it to replace the
10378 // field in RotatedOldVal.
10379 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal)
10380 .addReg(RotatedOldVal)
10381 .addReg(Src2.getReg())
10382 .addImm(32)
10383 .addImm(31 + BitSize)
10384 .addImm(32 - BitSize)
10385 .setOperandDead(6);
10386 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
10387 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
10388 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
10389 .addReg(OldVal)
10390 .addReg(NewVal)
10391 .add(Base)
10392 .addImm(Disp);
10393 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10395 MBB->addSuccessor(LoopMBB);
10396 MBB->addSuccessor(DoneMBB);
10397
10398 MI.eraseFromParent();
10399 return DoneMBB;
10400}
10401
10402// Implement EmitInstrWithCustomInserter for subword pseudo
10403// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
10404// instruction that should be used to compare the current field with the
10405// minimum or maximum value. KeepOldMask is the BRC condition-code mask
10406// for when the current field should be kept.
10407MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
10408 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
10409 unsigned KeepOldMask) const {
10410 MachineFunction &MF = *MBB->getParent();
10411 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10412 MachineRegisterInfo &MRI = MF.getRegInfo();
10413
10414 // Extract the operands. Base can be a register or a frame index.
10415 Register Dest = MI.getOperand(0).getReg();
10416 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
10417 int64_t Disp = MI.getOperand(2).getImm();
10418 Register Src2 = MI.getOperand(3).getReg();
10419 Register BitShift = MI.getOperand(4).getReg();
10420 Register NegBitShift = MI.getOperand(5).getReg();
10421 unsigned BitSize = MI.getOperand(6).getImm();
10422 DebugLoc DL = MI.getDebugLoc();
10423
10424 // Get the right opcodes for the displacement.
10425 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
10426 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
10427 assert(LOpcode && CSOpcode && "Displacement out of range");
10428
10429 // Create virtual registers for temporary results.
10430 Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10431 Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10432 Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10433 Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10434 Register RotatedAltVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10435 Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
10436
10437 // Insert 3 basic blocks for the loop.
10438 MachineBasicBlock *StartMBB = MBB;
10439 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10440 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
10441 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(LoopMBB);
10442 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(UseAltMBB);
10443
10444 // StartMBB:
10445 // ...
10446 // %OrigVal = L Disp(%Base)
10447 // # fall through to LoopMBB
10448 MBB = StartMBB;
10449 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
10450 MBB->addSuccessor(LoopMBB);
10451
10452 // LoopMBB:
10453 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
10454 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10455 // CompareOpcode %RotatedOldVal, %Src2
10456 // BRC KeepOldMask, UpdateMBB
10457 MBB = LoopMBB;
10458 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
10459 .addReg(OrigVal).addMBB(StartMBB)
10460 .addReg(Dest).addMBB(UpdateMBB);
10461 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
10462 .addReg(OldVal).addReg(BitShift).addImm(0);
10463 BuildMI(MBB, DL, TII->get(CompareOpcode))
10464 .addReg(RotatedOldVal).addReg(Src2);
10465 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10466 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB);
10467 MBB->addSuccessor(UpdateMBB);
10468 MBB->addSuccessor(UseAltMBB);
10469
10470 // UseAltMBB:
10471 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
10472 // # fall through to UpdateMBB
10473 MBB = UseAltMBB;
10474 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)
10475 .addReg(RotatedOldVal)
10476 .addReg(Src2)
10477 .addImm(32)
10478 .addImm(31 + BitSize)
10479 .addImm(0)
10480 .setOperandDead(6);
10481 MBB->addSuccessor(UpdateMBB);
10482
10483 // UpdateMBB:
10484 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
10485 // [ %RotatedAltVal, UseAltMBB ]
10486 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10487 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10488 // JNE LoopMBB
10489 // # fall through to DoneMBB
10490 MBB = UpdateMBB;
10491 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)
10492 .addReg(RotatedOldVal).addMBB(LoopMBB)
10493 .addReg(RotatedAltVal).addMBB(UseAltMBB);
10494 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
10495 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
10496 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
10497 .addReg(OldVal)
10498 .addReg(NewVal)
10499 .add(Base)
10500 .addImm(Disp);
10501 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10503 MBB->addSuccessor(LoopMBB);
10504 MBB->addSuccessor(DoneMBB);
10505
10506 MI.eraseFromParent();
10507 return DoneMBB;
10508}
10509
10510// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
10511// instruction MI.
10513SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
10514 MachineBasicBlock *MBB) const {
10515 MachineFunction &MF = *MBB->getParent();
10516 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10517 MachineRegisterInfo &MRI = MF.getRegInfo();
10518
10519 // Extract the operands. Base can be a register or a frame index.
10520 Register Dest = MI.getOperand(0).getReg();
10521 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
10522 int64_t Disp = MI.getOperand(2).getImm();
10523 Register CmpVal = MI.getOperand(3).getReg();
10524 Register OrigSwapVal = MI.getOperand(4).getReg();
10525 Register BitShift = MI.getOperand(5).getReg();
10526 Register NegBitShift = MI.getOperand(6).getReg();
10527 int64_t BitSize = MI.getOperand(7).getImm();
10528 DebugLoc DL = MI.getDebugLoc();
10529
10530 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
10531
10532 // Get the right opcodes for the displacement and zero-extension.
10533 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
10534 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
10535 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10536 assert(LOpcode && CSOpcode && "Displacement out of range");
10537
10538 // Create virtual registers for temporary results.
10539 Register OrigOldVal = MRI.createVirtualRegister(RC);
10540 Register OldVal = MRI.createVirtualRegister(RC);
10541 Register SwapVal = MRI.createVirtualRegister(RC);
10542 Register StoreVal = MRI.createVirtualRegister(RC);
10543 Register OldValRot = MRI.createVirtualRegister(RC);
10544 Register RetryOldVal = MRI.createVirtualRegister(RC);
10545 Register RetrySwapVal = MRI.createVirtualRegister(RC);
10546
10547 // Insert 2 basic blocks for the loop.
10548 MachineBasicBlock *StartMBB = MBB;
10549 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10550 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
10551 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(LoopMBB);
10552
10553 // StartMBB:
10554 // ...
10555 // %OrigOldVal = L Disp(%Base)
10556 // # fall through to LoopMBB
10557 MBB = StartMBB;
10558 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal)
10559 .add(Base)
10560 .addImm(Disp)
10561 .addReg(0);
10562 MBB->addSuccessor(LoopMBB);
10563
10564 // LoopMBB:
10565 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
10566 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
10567 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
10568 // ^^ The low BitSize bits contain the field
10569 // of interest.
10570 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
10571 // ^^ Replace the upper 32-BitSize bits of the
10572 // swap value with those that we loaded and rotated.
10573 // %Dest = LL[CH] %OldValRot
10574 // CR %Dest, %CmpVal
10575 // JNE DoneMBB
10576 // # Fall through to SetMBB
10577 MBB = LoopMBB;
10578 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
10579 .addReg(OrigOldVal).addMBB(StartMBB)
10580 .addReg(RetryOldVal).addMBB(SetMBB);
10581 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
10582 .addReg(OrigSwapVal).addMBB(StartMBB)
10583 .addReg(RetrySwapVal).addMBB(SetMBB);
10584 BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)
10585 .addReg(OldVal).addReg(BitShift).addImm(BitSize);
10586 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
10587 .addReg(SwapVal)
10588 .addReg(OldValRot)
10589 .addImm(32)
10590 .addImm(63 - BitSize)
10591 .addImm(0)
10592 .setOperandDead(6);
10593 BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)
10594 .addReg(OldValRot);
10595 BuildMI(MBB, DL, TII->get(SystemZ::CR))
10596 .addReg(Dest).addReg(CmpVal);
10597 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10600 MBB->addSuccessor(DoneMBB);
10601 MBB->addSuccessor(SetMBB);
10602
10603 // SetMBB:
10604 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
10605 // ^^ Rotate the new field to its proper position.
10606 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
10607 // JNE LoopMBB
10608 // # fall through to ExitMBB
10609 MBB = SetMBB;
10610 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
10611 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
10612 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
10613 .addReg(OldVal)
10614 .addReg(StoreVal)
10615 .add(Base)
10616 .addImm(Disp);
10617 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10619 MBB->addSuccessor(LoopMBB);
10620 MBB->addSuccessor(DoneMBB);
10621
10622 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
10623 // to the block after the loop. At this point, CC may have been defined
10624 // either by the CR in LoopMBB or by the CS in SetMBB.
10625 if (!MI.registerDefIsDead(SystemZ::CC, /*TRI=*/nullptr))
10626 DoneMBB->addLiveIn(SystemZ::CC);
10627
10628 MI.eraseFromParent();
10629 return DoneMBB;
10630}
10631
10633SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
10635 unsigned Opcode, bool IsMemset) const {
10636 MachineFunction &MF = *MBB->getParent();
10637 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10638 MachineRegisterInfo &MRI = MF.getRegInfo();
10639 DebugLoc DL = MI.getDebugLoc();
10640
10641 MachineOperand DestBase = earlyUseOperand(MI.getOperand(0));
10642 uint64_t DestDisp = MI.getOperand(1).getImm();
10643 MachineOperand SrcBase = MachineOperand::CreateReg(0U, false);
10644 uint64_t SrcDisp;
10645
10646 // Fold the displacement Disp if it is out of range.
10647 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
10648 if (!isUInt<12>(Disp)) {
10649 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
10650 unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);
10651 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opcode), Reg)
10652 .add(Base).addImm(Disp).addReg(0);
10654 Disp = 0;
10655 }
10656 };
10657
10658 if (!IsMemset) {
10659 SrcBase = earlyUseOperand(MI.getOperand(2));
10660 SrcDisp = MI.getOperand(3).getImm();
10661 } else {
10662 SrcBase = DestBase;
10663 SrcDisp = DestDisp++;
10664 foldDisplIfNeeded(DestBase, DestDisp);
10665 }
10666
10667 MachineOperand &LengthMO = MI.getOperand(IsMemset ? 2 : 4);
10668 bool IsImmForm = LengthMO.isImm();
10669 bool IsRegForm = !IsImmForm;
10670
10671 // Build and insert one Opcode of Length, with special treatment for memset.
10672 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10674 MachineOperand DBase, uint64_t DDisp,
10675 MachineOperand SBase, uint64_t SDisp,
10676 unsigned Length) -> void {
10677 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
10678 if (IsMemset) {
10679 MachineOperand ByteMO = earlyUseOperand(MI.getOperand(3));
10680 if (ByteMO.isImm())
10681 BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::MVI))
10682 .add(SBase).addImm(SDisp).add(ByteMO);
10683 else
10684 BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::STC))
10685 .add(ByteMO).add(SBase).addImm(SDisp).addReg(0);
10686 if (--Length == 0)
10687 return;
10688 }
10689 BuildMI(*MBB, InsPos, DL, TII->get(Opcode))
10690 .add(DBase).addImm(DDisp).addImm(Length)
10691 .add(SBase).addImm(SDisp)
10692 .setMemRefs(MI.memoperands());
10693 };
10694
10695 bool NeedsLoop = false;
10696 uint64_t ImmLength = 0;
10697 Register LenAdjReg = SystemZ::NoRegister;
10698 if (IsImmForm) {
10699 ImmLength = LengthMO.getImm();
10700 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
10701 if (ImmLength == 0) {
10702 MI.eraseFromParent();
10703 return MBB;
10704 }
10705 if (Opcode == SystemZ::CLC) {
10706 if (ImmLength > 3 * 256)
10707 // A two-CLC sequence is a clear win over a loop, not least because
10708 // it needs only one branch. A three-CLC sequence needs the same
10709 // number of branches as a loop (i.e. 2), but is shorter. That
10710 // brings us to lengths greater than 768 bytes. It seems relatively
10711 // likely that a difference will be found within the first 768 bytes,
10712 // so we just optimize for the smallest number of branch
10713 // instructions, in order to avoid polluting the prediction buffer
10714 // too much.
10715 NeedsLoop = true;
10716 } else if (ImmLength > 6 * 256)
10717 // The heuristic we use is to prefer loops for anything that would
10718 // require 7 or more MVCs. With these kinds of sizes there isn't much
10719 // to choose between straight-line code and looping code, since the
10720 // time will be dominated by the MVCs themselves.
10721 NeedsLoop = true;
10722 } else {
10723 NeedsLoop = true;
10724 LenAdjReg = LengthMO.getReg();
10725 }
10726
10727 // When generating more than one CLC, all but the last will need to
10728 // branch to the end when a difference is found.
10729 MachineBasicBlock *EndMBB =
10730 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10732 : nullptr);
10733
10734 if (NeedsLoop) {
10735 Register StartCountReg =
10736 MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
10737 if (IsImmForm) {
10738 TII->loadImmediate(*MBB, MI, StartCountReg, ImmLength / 256);
10739 ImmLength &= 255;
10740 } else {
10741 BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg)
10742 .addReg(LenAdjReg)
10743 .addReg(0)
10744 .addImm(8);
10745 }
10746
10747 bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
10748 auto loadZeroAddress = [&]() -> MachineOperand {
10749 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
10750 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0);
10751 return MachineOperand::CreateReg(Reg, false);
10752 };
10753 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
10754 DestBase = loadZeroAddress();
10755 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
10756 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10757
10758 MachineBasicBlock *StartMBB = nullptr;
10759 MachineBasicBlock *LoopMBB = nullptr;
10760 MachineBasicBlock *NextMBB = nullptr;
10761 MachineBasicBlock *DoneMBB = nullptr;
10762 MachineBasicBlock *AllDoneMBB = nullptr;
10763
10764 Register StartSrcReg = forceReg(MI, SrcBase, TII);
10765 Register StartDestReg =
10766 (HaveSingleBase ? StartSrcReg : forceReg(MI, DestBase, TII));
10767
10768 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
10769 Register ThisSrcReg = MRI.createVirtualRegister(RC);
10770 Register ThisDestReg =
10771 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RC));
10772 Register NextSrcReg = MRI.createVirtualRegister(RC);
10773 Register NextDestReg =
10774 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RC));
10775 RC = &SystemZ::GR64BitRegClass;
10776 Register ThisCountReg = MRI.createVirtualRegister(RC);
10777 Register NextCountReg = MRI.createVirtualRegister(RC);
10778
10779 if (IsRegForm) {
10780 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10781 StartMBB = SystemZ::emitBlockAfter(MBB);
10782 LoopMBB = SystemZ::emitBlockAfter(StartMBB);
10783 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
10784 DoneMBB = SystemZ::emitBlockAfter(NextMBB);
10785
10786 // MBB:
10787 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
10788 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
10789 .addReg(LenAdjReg).addImm(IsMemset ? -2 : -1);
10790 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10792 .addMBB(AllDoneMBB);
10793 MBB->addSuccessor(AllDoneMBB);
10794 if (!IsMemset)
10795 MBB->addSuccessor(StartMBB);
10796 else {
10797 // MemsetOneCheckMBB:
10798 // # Jump to MemsetOneMBB for a memset of length 1, or
10799 // # fall thru to StartMBB.
10800 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
10801 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(&*MF.rbegin());
10802 MBB->addSuccessor(MemsetOneCheckMBB);
10803 MBB = MemsetOneCheckMBB;
10804 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
10805 .addReg(LenAdjReg).addImm(-1);
10806 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10808 .addMBB(MemsetOneMBB);
10809 MBB->addSuccessor(MemsetOneMBB, {10, 100});
10810 MBB->addSuccessor(StartMBB, {90, 100});
10811
10812 // MemsetOneMBB:
10813 // # Jump back to AllDoneMBB after a single MVI or STC.
10814 MBB = MemsetOneMBB;
10815 insertMemMemOp(MBB, MBB->end(),
10816 MachineOperand::CreateReg(StartDestReg, false), DestDisp,
10817 MachineOperand::CreateReg(StartSrcReg, false), SrcDisp,
10818 1);
10819 BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(AllDoneMBB);
10820 MBB->addSuccessor(AllDoneMBB);
10821 }
10822
10823 // StartMBB:
10824 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
10825 MBB = StartMBB;
10826 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
10827 .addReg(StartCountReg).addImm(0);
10828 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10830 .addMBB(DoneMBB);
10831 MBB->addSuccessor(DoneMBB);
10832 MBB->addSuccessor(LoopMBB);
10833 }
10834 else {
10835 StartMBB = MBB;
10836 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10837 LoopMBB = SystemZ::emitBlockAfter(StartMBB);
10838 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
10839
10840 // StartMBB:
10841 // # fall through to LoopMBB
10842 MBB->addSuccessor(LoopMBB);
10843
10844 DestBase = MachineOperand::CreateReg(NextDestReg, false);
10845 SrcBase = MachineOperand::CreateReg(NextSrcReg, false);
10846 if (EndMBB && !ImmLength)
10847 // If the loop handled the whole CLC range, DoneMBB will be empty with
10848 // CC live-through into EndMBB, so add it as live-in.
10849 DoneMBB->addLiveIn(SystemZ::CC);
10850 }
10851
10852 // LoopMBB:
10853 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
10854 // [ %NextDestReg, NextMBB ]
10855 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
10856 // [ %NextSrcReg, NextMBB ]
10857 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
10858 // [ %NextCountReg, NextMBB ]
10859 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
10860 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
10861 // ( JLH EndMBB )
10862 //
10863 // The prefetch is used only for MVC. The JLH is used only for CLC.
10864 MBB = LoopMBB;
10865 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg)
10866 .addReg(StartDestReg).addMBB(StartMBB)
10867 .addReg(NextDestReg).addMBB(NextMBB);
10868 if (!HaveSingleBase)
10869 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg)
10870 .addReg(StartSrcReg).addMBB(StartMBB)
10871 .addReg(NextSrcReg).addMBB(NextMBB);
10872 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg)
10873 .addReg(StartCountReg).addMBB(StartMBB)
10874 .addReg(NextCountReg).addMBB(NextMBB);
10875 if (Opcode == SystemZ::MVC)
10876 BuildMI(MBB, DL, TII->get(SystemZ::PFD))
10878 .addReg(ThisDestReg).addImm(DestDisp - IsMemset + 768).addReg(0);
10879 insertMemMemOp(MBB, MBB->end(),
10880 MachineOperand::CreateReg(ThisDestReg, false), DestDisp,
10881 MachineOperand::CreateReg(ThisSrcReg, false), SrcDisp, 256);
10882 if (EndMBB) {
10883 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10885 .addMBB(EndMBB);
10886 MBB->addSuccessor(EndMBB);
10887 MBB->addSuccessor(NextMBB);
10888 }
10889
10890 // NextMBB:
10891 // %NextDestReg = LA 256(%ThisDestReg)
10892 // %NextSrcReg = LA 256(%ThisSrcReg)
10893 // %NextCountReg = AGHI %ThisCountReg, -1
10894 // CGHI %NextCountReg, 0
10895 // JLH LoopMBB
10896 // # fall through to DoneMBB
10897 //
10898 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
10899 MBB = NextMBB;
10900 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg)
10901 .addReg(ThisDestReg).addImm(256).addReg(0);
10902 if (!HaveSingleBase)
10903 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg)
10904 .addReg(ThisSrcReg).addImm(256).addReg(0);
10905 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg)
10906 .addReg(ThisCountReg)
10907 .addImm(-1)
10908 .setOperandDead(3);
10909 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
10910 .addReg(NextCountReg).addImm(0);
10911 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10913 .addMBB(LoopMBB);
10914 MBB->addSuccessor(LoopMBB);
10915 MBB->addSuccessor(DoneMBB);
10916
10917 MBB = DoneMBB;
10918 if (IsRegForm) {
10919 // DoneMBB:
10920 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
10921 // # Use EXecute Relative Long for the remainder of the bytes. The target
10922 // instruction of the EXRL will have a length field of 1 since 0 is an
10923 // illegal value. The number of bytes processed becomes (%LenAdjReg &
10924 // 0xff) + 1.
10925 // # Fall through to AllDoneMBB.
10926 Register RemSrcReg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
10927 Register RemDestReg = HaveSingleBase ? RemSrcReg
10928 : MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
10929 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg)
10930 .addReg(StartDestReg).addMBB(StartMBB)
10931 .addReg(NextDestReg).addMBB(NextMBB);
10932 if (!HaveSingleBase)
10933 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)
10934 .addReg(StartSrcReg).addMBB(StartMBB)
10935 .addReg(NextSrcReg).addMBB(NextMBB);
10936 if (IsMemset)
10937 insertMemMemOp(MBB, MBB->end(),
10938 MachineOperand::CreateReg(RemDestReg, false), DestDisp,
10939 MachineOperand::CreateReg(RemSrcReg, false), SrcDisp, 1);
10940 MachineInstrBuilder EXRL_MIB =
10941 BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))
10942 .addImm(Opcode)
10943 .addReg(LenAdjReg)
10944 .addReg(RemDestReg).addImm(DestDisp)
10945 .addReg(RemSrcReg).addImm(SrcDisp);
10946 MBB->addSuccessor(AllDoneMBB);
10947 MBB = AllDoneMBB;
10948 if (Opcode != SystemZ::MVC) {
10949 EXRL_MIB.addReg(SystemZ::CC, RegState::ImplicitDefine);
10950 if (EndMBB)
10951 MBB->addLiveIn(SystemZ::CC);
10952 }
10953 }
10954 MF.getProperties().resetNoPHIs();
10955 }
10956
10957 // Handle any remaining bytes with straight-line code.
10958 while (ImmLength > 0) {
10959 uint64_t ThisLength = std::min(ImmLength, uint64_t(256));
10960 // The previous iteration might have created out-of-range displacements.
10961 // Apply them using LA/LAY if so.
10962 foldDisplIfNeeded(DestBase, DestDisp);
10963 foldDisplIfNeeded(SrcBase, SrcDisp);
10964 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10965 DestDisp += ThisLength;
10966 SrcDisp += ThisLength;
10967 ImmLength -= ThisLength;
10968 // If there's another CLC to go, branch to the end if a difference
10969 // was found.
10970 if (EndMBB && ImmLength > 0) {
10971 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
10972 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
10974 .addMBB(EndMBB);
10975 MBB->addSuccessor(EndMBB);
10976 MBB->addSuccessor(NextMBB);
10977 MBB = NextMBB;
10978 }
10979 }
10980 if (EndMBB) {
10981 MBB->addSuccessor(EndMBB);
10982 MBB = EndMBB;
10983 MBB->addLiveIn(SystemZ::CC);
10984 }
10985
10986 MI.eraseFromParent();
10987 return MBB;
10988}
10989
10991SystemZTargetLowering::emitMemmoveImm(MachineInstr &MI,
10992 MachineBasicBlock *MBB) const {
10993 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10994
10995 DebugLoc DL = MI.getDebugLoc();
10996 MachineOperand DstAddr = earlyUseOperand(MI.getOperand(0));
10997 MachineOperand SrcAddr = earlyUseOperand(MI.getOperand(1));
10998 uint64_t Len = MI.getOperand(2).getImm();
10999 assert(Len > 0 && Len <= 256 && "Memmove of of unsupported constant length.");
11000
11001 // Use MVC or MVCRL after comparing the addresses.
11002 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
11003 MachineBasicBlock *MvcMBB = SystemZ::emitBlockAfter(MBB);
11004 MachineBasicBlock *MvcrlMBB = SystemZ::emitBlockAfter(MvcMBB);
11005 MBB->addSuccessor(MvcMBB);
11006 MBB->addSuccessor(MvcrlMBB);
11007 MvcMBB->addSuccessor(DoneMBB);
11008 MvcrlMBB->addSuccessor(DoneMBB);
11009
11010 BuildMI(MBB, DL, TII->get(SystemZ::CLGR)).add(SrcAddr).add(DstAddr);
11011 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
11013 .addMBB(MvcrlMBB);
11014
11015 BuildMI(MvcMBB, DL, TII->get(SystemZ::MVC))
11016 .add(DstAddr).addImm(0)
11017 .addImm(Len)
11018 .add(SrcAddr).addImm(0)
11019 .setMemRefs(MI.memoperands());
11020 BuildMI(MvcMBB, DL, TII->get(SystemZ::J)).addMBB(DoneMBB);
11021
11022 BuildMI(MvcrlMBB, DL, TII->get(SystemZ::LHI), SystemZ::R0L).addImm(Len - 1);
11023 BuildMI(MvcrlMBB, DL, TII->get(SystemZ::MVCRL))
11024 .add(DstAddr).addImm(0)
11025 .add(SrcAddr).addImm(0)
11026 .setMemRefs(MI.memoperands());
11027
11028 MI.eraseFromParent();
11029 return DoneMBB;
11030}
11031
11032// Decompose string pseudo-instruction MI into a loop that continually performs
11033// Opcode until CC != 3.
11034MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
11035 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
11036 MachineFunction &MF = *MBB->getParent();
11037 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11038 MachineRegisterInfo &MRI = MF.getRegInfo();
11039 DebugLoc DL = MI.getDebugLoc();
11040
11041 uint64_t End1Reg = MI.getOperand(0).getReg();
11042 uint64_t Start1Reg = MI.getOperand(1).getReg();
11043 uint64_t Start2Reg = MI.getOperand(2).getReg();
11044 uint64_t CharReg = MI.getOperand(3).getReg();
11045
11046 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
11047 uint64_t This1Reg = MRI.createVirtualRegister(RC);
11048 uint64_t This2Reg = MRI.createVirtualRegister(RC);
11049 uint64_t End2Reg = MRI.createVirtualRegister(RC);
11050
11051 MachineBasicBlock *StartMBB = MBB;
11052 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
11053 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
11054
11055 // StartMBB:
11056 // # fall through to LoopMBB
11057 MBB->addSuccessor(LoopMBB);
11058
11059 // LoopMBB:
11060 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
11061 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
11062 // R0L = %CharReg
11063 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
11064 // JO LoopMBB
11065 // # fall through to DoneMBB
11066 //
11067 // The load of R0L can be hoisted by post-RA LICM.
11068 MBB = LoopMBB;
11069
11070 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg)
11071 .addReg(Start1Reg).addMBB(StartMBB)
11072 .addReg(End1Reg).addMBB(LoopMBB);
11073 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg)
11074 .addReg(Start2Reg).addMBB(StartMBB)
11075 .addReg(End2Reg).addMBB(LoopMBB);
11076 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg);
11077 BuildMI(MBB, DL, TII->get(Opcode))
11078 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define)
11079 .addReg(This1Reg).addReg(This2Reg);
11080 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
11082 MBB->addSuccessor(LoopMBB);
11083 MBB->addSuccessor(DoneMBB);
11084
11085 DoneMBB->addLiveIn(SystemZ::CC);
11086
11087 MI.eraseFromParent();
11088 return DoneMBB;
11089}
11090
11091// Update TBEGIN instruction with final opcode and register clobbers.
11092MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
11093 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
11094 bool NoFloat) const {
11095 MachineFunction &MF = *MBB->getParent();
11096 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
11097 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11098
11099 // Update opcode.
11100 MI.setDesc(TII->get(Opcode));
11101
11102 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
11103 // Make sure to add the corresponding GRSM bits if they are missing.
11104 uint64_t Control = MI.getOperand(2).getImm();
11105 static const unsigned GPRControlBit[16] = {
11106 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
11107 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
11108 };
11109 Control |= GPRControlBit[15];
11110 if (TFI->hasFP(MF))
11111 Control |= GPRControlBit[11];
11112 MI.getOperand(2).setImm(Control);
11113
11114 // Add GPR clobbers.
11115 for (int I = 0; I < 16; I++) {
11116 if ((Control & GPRControlBit[I]) == 0) {
11117 unsigned Reg = SystemZMC::GR64Regs[I];
11118 MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
11119 }
11120 }
11121
11122 // Add FPR/VR clobbers.
11123 if (!NoFloat && (Control & 4) != 0) {
11124 if (Subtarget.hasVector()) {
11125 for (unsigned Reg : SystemZMC::VR128Regs) {
11126 MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
11127 }
11128 } else {
11129 for (unsigned Reg : SystemZMC::FP64Regs) {
11130 MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
11131 }
11132 }
11133 }
11134
11135 return MBB;
11136}
11137
11138MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
11139 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
11140 MachineFunction &MF = *MBB->getParent();
11141 MachineRegisterInfo *MRI = &MF.getRegInfo();
11142 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11143 DebugLoc DL = MI.getDebugLoc();
11144
11145 Register SrcReg = MI.getOperand(0).getReg();
11146
11147 // Create new virtual register of the same class as source.
11148 const TargetRegisterClass *RC = MRI->getRegClass(SrcReg);
11149 Register DstReg = MRI->createVirtualRegister(RC);
11150
11151 // Replace pseudo with a normal load-and-test that models the def as
11152 // well.
11153 BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg)
11154 .addReg(SrcReg)
11155 .setMIFlags(MI.getFlags());
11156 MI.eraseFromParent();
11157
11158 return MBB;
11159}
11160
11161MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
11163 MachineFunction &MF = *MBB->getParent();
11164 MachineRegisterInfo *MRI = &MF.getRegInfo();
11165 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11166 DebugLoc DL = MI.getDebugLoc();
11167 const unsigned ProbeSize = getStackProbeSize(MF);
11168 Register DstReg = MI.getOperand(0).getReg();
11169 Register SizeReg = MI.getOperand(2).getReg();
11170
11171 MachineBasicBlock *StartMBB = MBB;
11172 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
11173 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(StartMBB);
11174 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(LoopTestMBB);
11175 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(LoopBodyMBB);
11176 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(TailTestMBB);
11177
11178 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(MachinePointerInfo(),
11180
11181 Register PHIReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
11182 Register IncReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
11183
11184 // LoopTestMBB
11185 // BRC TailTestMBB
11186 // # fallthrough to LoopBodyMBB
11187 StartMBB->addSuccessor(LoopTestMBB);
11188 MBB = LoopTestMBB;
11189 BuildMI(MBB, DL, TII->get(SystemZ::PHI), PHIReg)
11190 .addReg(SizeReg)
11191 .addMBB(StartMBB)
11192 .addReg(IncReg)
11193 .addMBB(LoopBodyMBB);
11194 BuildMI(MBB, DL, TII->get(SystemZ::CLGFI))
11195 .addReg(PHIReg)
11196 .addImm(ProbeSize);
11197 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
11199 .addMBB(TailTestMBB);
11200 MBB->addSuccessor(LoopBodyMBB);
11201 MBB->addSuccessor(TailTestMBB);
11202
11203 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
11204 // J LoopTestMBB
11205 MBB = LoopBodyMBB;
11206 BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), IncReg)
11207 .addReg(PHIReg)
11208 .addImm(ProbeSize)
11209 .setOperandDead(3);
11210 BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), SystemZ::R15D)
11211 .addReg(SystemZ::R15D)
11212 .addImm(ProbeSize)
11213 .setOperandDead(3);
11214 BuildMI(MBB, DL, TII->get(SystemZ::CG))
11215 .addReg(SystemZ::R15D)
11216 .addReg(SystemZ::R15D)
11217 .addImm(ProbeSize - 8)
11218 .addReg(0)
11219 .setOperandDead(4)
11220 .setMemRefs(VolLdMMO);
11221 BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(LoopTestMBB);
11222 MBB->addSuccessor(LoopTestMBB);
11223
11224 // TailTestMBB
11225 // BRC DoneMBB
11226 // # fallthrough to TailMBB
11227 MBB = TailTestMBB;
11228 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
11229 .addReg(PHIReg)
11230 .addImm(0);
11231 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
11233 .addMBB(DoneMBB);
11234 MBB->addSuccessor(TailMBB);
11235 MBB->addSuccessor(DoneMBB);
11236
11237 // TailMBB
11238 // # fallthrough to DoneMBB
11239 MBB = TailMBB;
11240 BuildMI(MBB, DL, TII->get(SystemZ::SLGR), SystemZ::R15D)
11241 .addReg(SystemZ::R15D)
11242 .addReg(PHIReg)
11243 .setOperandDead(3);
11244 BuildMI(MBB, DL, TII->get(SystemZ::CG))
11245 .addReg(SystemZ::R15D)
11246 .addReg(SystemZ::R15D)
11247 .addImm(-8)
11248 .addReg(PHIReg)
11249 .setOperandDead(4)
11250 .setMemRefs(VolLdMMO);
11251 MBB->addSuccessor(DoneMBB);
11252
11253 // DoneMBB
11254 MBB = DoneMBB;
11255 BuildMI(*MBB, MBB->begin(), DL, TII->get(TargetOpcode::COPY), DstReg)
11256 .addReg(SystemZ::R15D);
11257
11258 MI.eraseFromParent();
11259 return DoneMBB;
11260}
11261
11262SDValue SystemZTargetLowering::
11263getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
11265 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
11266 SDLoc DL(SP);
11267 return DAG.getNode(ISD::ADD, DL, MVT::i64, SP,
11268 DAG.getIntPtrConstant(TFL->getBackchainOffset(MF), DL));
11269}
11270
11271// Replace a _STACKGUARD_DAG pseudo with a _STACKGUARD pseudo, adding
11272// a dead early-clobber def reg that will be used as a scratch register
11273// when the pseudo is expanded.
11274MachineBasicBlock *SystemZTargetLowering::emitStackGuardPseudo(
11275 MachineInstr &MI, MachineBasicBlock *MBB, unsigned PseudoOp) const {
11276 MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
11277 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11278 DebugLoc DL = MI.getDebugLoc();
11279 Register AddrReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
11280 BuildMI(*MBB, MI, DL, TII->get(PseudoOp), AddrReg)
11281 .addFrameIndex(MI.getOperand(0).getIndex())
11282 .addImm(MI.getOperand(1).getImm());
11283 MI.eraseFromParent();
11284 return MBB;
11285}
11286
11289 switch (MI.getOpcode()) {
11290 case SystemZ::ADJCALLSTACKDOWN:
11291 case SystemZ::ADJCALLSTACKUP:
11292 return emitAdjCallStack(MI, MBB);
11293
11294 case SystemZ::Select32:
11295 case SystemZ::Select64:
11296 case SystemZ::Select128:
11297 case SystemZ::SelectF32:
11298 case SystemZ::SelectF64:
11299 case SystemZ::SelectF128:
11300 case SystemZ::SelectVR32:
11301 case SystemZ::SelectVR64:
11302 case SystemZ::SelectVR128:
11303 return emitSelect(MI, MBB);
11304
11305 case SystemZ::CondStore8Mux:
11306 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false);
11307 case SystemZ::CondStore8MuxInv:
11308 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true);
11309 case SystemZ::CondStore16Mux:
11310 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false);
11311 case SystemZ::CondStore16MuxInv:
11312 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true);
11313 case SystemZ::CondStore32Mux:
11314 return emitCondStore(MI, MBB, SystemZ::STMux, SystemZ::STOCMux, false);
11315 case SystemZ::CondStore32MuxInv:
11316 return emitCondStore(MI, MBB, SystemZ::STMux, SystemZ::STOCMux, true);
11317 case SystemZ::CondStore8:
11318 return emitCondStore(MI, MBB, SystemZ::STC, 0, false);
11319 case SystemZ::CondStore8Inv:
11320 return emitCondStore(MI, MBB, SystemZ::STC, 0, true);
11321 case SystemZ::CondStore16:
11322 return emitCondStore(MI, MBB, SystemZ::STH, 0, false);
11323 case SystemZ::CondStore16Inv:
11324 return emitCondStore(MI, MBB, SystemZ::STH, 0, true);
11325 case SystemZ::CondStore32:
11326 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false);
11327 case SystemZ::CondStore32Inv:
11328 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true);
11329 case SystemZ::CondStore64:
11330 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false);
11331 case SystemZ::CondStore64Inv:
11332 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true);
11333 case SystemZ::CondStoreF32:
11334 return emitCondStore(MI, MBB, SystemZ::STE, 0, false);
11335 case SystemZ::CondStoreF32Inv:
11336 return emitCondStore(MI, MBB, SystemZ::STE, 0, true);
11337 case SystemZ::CondStoreF64:
11338 return emitCondStore(MI, MBB, SystemZ::STD, 0, false);
11339 case SystemZ::CondStoreF64Inv:
11340 return emitCondStore(MI, MBB, SystemZ::STD, 0, true);
11341
11342 case SystemZ::SCmp128Hi:
11343 return emitICmp128Hi(MI, MBB, false);
11344 case SystemZ::UCmp128Hi:
11345 return emitICmp128Hi(MI, MBB, true);
11346
11347
11348 case SystemZ::ATOMIC_SWAPW:
11349 return emitAtomicLoadBinary(MI, MBB, 0);
11350
11351 case SystemZ::ATOMIC_LOADW_AR:
11352 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR);
11353 case SystemZ::ATOMIC_LOADW_AFI:
11354 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI);
11355
11356 case SystemZ::ATOMIC_LOADW_SR:
11357 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR);
11358
11359 case SystemZ::ATOMIC_LOADW_NR:
11360 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR);
11361 case SystemZ::ATOMIC_LOADW_NILH:
11362 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH);
11363
11364 case SystemZ::ATOMIC_LOADW_OR:
11365 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR);
11366 case SystemZ::ATOMIC_LOADW_OILH:
11367 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH);
11368
11369 case SystemZ::ATOMIC_LOADW_XR:
11370 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR);
11371 case SystemZ::ATOMIC_LOADW_XILF:
11372 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF);
11373
11374 case SystemZ::ATOMIC_LOADW_NRi:
11375 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, true);
11376 case SystemZ::ATOMIC_LOADW_NILHi:
11377 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, true);
11378
11379 case SystemZ::ATOMIC_LOADW_MIN:
11380 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_LE);
11381 case SystemZ::ATOMIC_LOADW_MAX:
11382 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_GE);
11383 case SystemZ::ATOMIC_LOADW_UMIN:
11384 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_LE);
11385 case SystemZ::ATOMIC_LOADW_UMAX:
11386 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_GE);
11387
11388 case SystemZ::ATOMIC_CMP_SWAPW:
11389 return emitAtomicCmpSwapW(MI, MBB);
11390 case SystemZ::MVCImm:
11391 case SystemZ::MVCReg:
11392 return emitMemMemWrapper(MI, MBB, SystemZ::MVC);
11393 case SystemZ::NCImm:
11394 return emitMemMemWrapper(MI, MBB, SystemZ::NC);
11395 case SystemZ::OCImm:
11396 return emitMemMemWrapper(MI, MBB, SystemZ::OC);
11397 case SystemZ::XCImm:
11398 case SystemZ::XCReg:
11399 return emitMemMemWrapper(MI, MBB, SystemZ::XC);
11400 case SystemZ::CLCImm:
11401 case SystemZ::CLCReg:
11402 return emitMemMemWrapper(MI, MBB, SystemZ::CLC);
11403 case SystemZ::MemsetImmImm:
11404 case SystemZ::MemsetImmReg:
11405 case SystemZ::MemsetRegImm:
11406 case SystemZ::MemsetRegReg:
11407 return emitMemMemWrapper(MI, MBB, SystemZ::MVC, true/*IsMemset*/);
11408 case SystemZ::MemmoveImm:
11409 return emitMemmoveImm(MI, MBB);
11410 case SystemZ::CLSTLoop:
11411 return emitStringWrapper(MI, MBB, SystemZ::CLST);
11412 case SystemZ::MVSTLoop:
11413 return emitStringWrapper(MI, MBB, SystemZ::MVST);
11414 case SystemZ::SRSTLoop:
11415 return emitStringWrapper(MI, MBB, SystemZ::SRST);
11416 case SystemZ::TBEGIN:
11417 return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, false);
11418 case SystemZ::TBEGIN_nofloat:
11419 return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, true);
11420 case SystemZ::TBEGINC:
11421 return emitTransactionBegin(MI, MBB, SystemZ::TBEGINC, true);
11422 case SystemZ::LTEBRCompare_Pseudo:
11423 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTEBR);
11424 case SystemZ::LTDBRCompare_Pseudo:
11425 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTDBR);
11426 case SystemZ::LTXBRCompare_Pseudo:
11427 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTXBR);
11428
11429 case SystemZ::PROBED_ALLOCA:
11430 return emitProbedAlloca(MI, MBB);
11431 case SystemZ::EH_SjLj_SetJmp:
11432 return emitEHSjLjSetJmp(MI, MBB);
11433 case SystemZ::EH_SjLj_LongJmp:
11434 return emitEHSjLjLongJmp(MI, MBB);
11435
11436 case TargetOpcode::STACKMAP:
11437 case TargetOpcode::PATCHPOINT:
11438 return emitPatchPoint(MI, MBB);
11439
11440 case SystemZ::MOV_STACKGUARD_DAG:
11441 return emitStackGuardPseudo(MI, MBB, SystemZ::MOV_STACKGUARD);
11442
11443 case SystemZ::CMP_STACKGUARD_DAG:
11444 return emitStackGuardPseudo(MI, MBB, SystemZ::CMP_STACKGUARD);
11445
11446 default:
11447 llvm_unreachable("Unexpected instr type to insert");
11448 }
11449}
11450
11451SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
11452 SelectionDAG &DAG) const {
11453 SDLoc dl(Op);
11454 /*
11455 The rounding method is in FPC Byte 3 bits 6-7, and has the following
11456 settings:
11457 00 Round to nearest
11458 01 Round to 0
11459 10 Round to +inf
11460 11 Round to -inf
11461
11462 FLT_ROUNDS, on the other hand, expects the following:
11463 -1 Undefined
11464 0 Round to 0
11465 1 Round to nearest
11466 2 Round to +inf
11467 3 Round to -inf
11468 */
11469
11470 // Save FPC to register.
11471 SDValue Chain = Op.getOperand(0);
11472 SDValue EFPC(
11473 DAG.getMachineNode(SystemZ::EFPC, dl, {MVT::i32, MVT::Other}, Chain), 0);
11474 Chain = EFPC.getValue(1);
11475
11476 // Transform as necessary
11477 SDValue CWD1 = DAG.getNode(ISD::AND, dl, MVT::i32, EFPC,
11478 DAG.getConstant(3, dl, MVT::i32));
11479 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
11480 SDValue CWD2 = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1,
11481 DAG.getNode(ISD::SRL, dl, MVT::i32, CWD1,
11482 DAG.getConstant(1, dl, MVT::i32)));
11483
11484 SDValue RetVal = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD2,
11485 DAG.getConstant(1, dl, MVT::i32));
11486 RetVal = DAG.getZExtOrTrunc(RetVal, dl, Op.getValueType());
11487
11488 return DAG.getMergeValues({RetVal, Chain}, dl);
11489}
11490
11491SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
11492 SelectionDAG &DAG) const {
11493 EVT VT = Op.getValueType();
11494 Op = Op.getOperand(0);
11495 EVT OpVT = Op.getValueType();
11496
11497 assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
11498
11499 SDLoc DL(Op);
11500
11501 // load a 0 vector for the third operand of VSUM.
11502 SDValue Zero = DAG.getSplatBuildVector(OpVT, DL, DAG.getConstant(0, DL, VT));
11503
11504 // execute VSUM.
11505 switch (OpVT.getScalarSizeInBits()) {
11506 case 8:
11507 case 16:
11508 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Zero);
11509 [[fallthrough]];
11510 case 32:
11511 case 64:
11512 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::i128, Op,
11513 DAG.getBitcast(Op.getValueType(), Zero));
11514 break;
11515 case 128:
11516 break; // VSUM over v1i128 should not happen and would be a noop
11517 default:
11518 llvm_unreachable("Unexpected scalar size.");
11519 }
11520 // Cast to original vector type, retrieve last element.
11521 return DAG.getNode(
11522 ISD::EXTRACT_VECTOR_ELT, DL, VT, DAG.getBitcast(OpVT, Op),
11523 DAG.getConstant(OpVT.getVectorNumElements() - 1, DL, MVT::i32));
11524}
11525
11526static void printFunctionArgExts(const Function *F, raw_ostream &OS) {
11527 FunctionType *FT = F->getFunctionType();
11528 const AttributeList &Attrs = F->getAttributes();
11529 if (Attrs.hasRetAttrs())
11530 OS << Attrs.getAsString(AttributeList::ReturnIndex) << " ";
11531 OS << *F->getReturnType() << " @" << F->getName() << "(";
11532 for (unsigned I = 0, E = FT->getNumParams(); I != E; ++I) {
11533 if (I)
11534 OS << ", ";
11535 OS << *FT->getParamType(I);
11536 AttributeSet ArgAttrs = Attrs.getParamAttrs(I);
11537 for (auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11538 if (ArgAttrs.hasAttribute(A))
11539 OS << " " << Attribute::getNameFromAttrKind(A);
11540 }
11541 if (F->isVarArg())
11542 OS << ", ...";
11543 OS << ")\n";
11544}
11545
11546bool SystemZTargetLowering::isInternal(const Function *Fn) const {
11547 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(Fn);
11548 if (Itr == IsInternalCache.end())
11549 Itr = IsInternalCache
11550 .insert(std::pair<const Function *, bool>(
11551 Fn, (Fn->hasLocalLinkage() && !Fn->hasAddressTaken())))
11552 .first;
11553 return Itr->second;
11554}
11555
11556bool SystemZTargetLowering::enableNarrowIntArgsVerification() const {
11557 if (!Subtarget.isTargetELF())
11558 return false;
11559
11561 return EnableIntArgExtCheck;
11562
11563#ifdef NDEBUG
11564 return false;
11565#endif
11566
11568}
11569
11570void SystemZTargetLowering::verifyNarrowIntegerArgs_Call(
11571 CallLoweringInfo &CLI, const SmallVectorImpl<ISD::OutputArg> &Outs,
11572 const Function *F, SDValue Callee) const {
11573 if (!enableNarrowIntArgsVerification())
11574 return;
11575 bool IsInternal = false;
11576 const Function *CalleeFn = nullptr;
11577 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee))
11578 if ((CalleeFn = dyn_cast<Function>(G->getGlobal())))
11579 IsInternal = isInternal(CalleeFn);
11580 bool C_ABI = CLI.CallConv == CallingConv::C && !IsInternal;
11581 if (!verifyNarrowIntegerArgs(C_ABI, Outs, CalleeFn, /*IsRet=*/false)) {
11582 errs() << "Callee: ";
11583 if (CalleeFn != nullptr)
11584 printFunctionArgExts(CalleeFn, errs());
11585 else
11586 errs() << "-\n";
11587 errs() << "Caller: ";
11589 if (CLI.CB)
11590 errs() << " " << *CLI.CB;
11591 llvm_unreachable("");
11592 }
11593 LLVM_DEBUG(dbgs() << "Outgoing call arguments verified as ABI compliant: ";
11594 if (CalleeFn != nullptr) printFunctionArgExts(CalleeFn, dbgs());
11595 else dbgs() << "-\n";);
11596}
11597
11598void SystemZTargetLowering::verifyNarrowIntegerArgs_Ret(
11600 const Function *F) const {
11601 if (!enableNarrowIntArgsVerification())
11602 return;
11603 bool C_ABI = CallConv == CallingConv::C && !isInternal(F);
11604 if (!verifyNarrowIntegerArgs(C_ABI, Outs, F, /*IsRet=*/true)) {
11605 errs() << "Returning from function: ";
11607 llvm_unreachable("");
11608 }
11609 LLVM_DEBUG(dbgs() << "Return argument verified as ABI compliant : ";
11611}
11612
11613// Verify that narrow integer arguments are extended according to the
11614// function header (calllee for a call, containing function for a ret).
11615// If C_ABI is true, an extension attribute must be present.
11616bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11617 bool C_ABI, const SmallVectorImpl<ISD::OutputArg> &Outs,
11618 const Function *FunHeader, bool IsRet) const {
11619 if (IsRet && Outs.size() != 1)
11620 return true;
11621
11622 for (unsigned i = 0; i < Outs.size(); ++i) {
11623 EVT OrigArgVT = Outs[i].ArgVT;
11624 if (!OrigArgVT.isScalarInteger() || OrigArgVT.getSizeInBits() >= 64)
11625 continue;
11626 assert(Outs[i].VT == MVT::i32 && "Unexpected integer argument VT.");
11627 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11628 unsigned OrigIdx = Outs[i].OrigArgIndex;
11629
11630 // Per the C ABI there must be an extension.
11631 if (C_ABI && !Flags.isSExt() && !Flags.isZExt() && !Flags.isNoExt()) {
11632 errs() << "ERROR: (C ABI violation) missing extension attribute on "
11633 << (IsRet ? "return value" : Twine("arg ") + Twine(OrigIdx))
11634 << ".\n";
11635 return false;
11636 }
11637
11638 // Flags should reflect the attributes of the function header.
11639 if (FunHeader) {
11640 const AttributeList &FunAttrs = FunHeader->getAttributes();
11641 const AttributeSet Attrs =
11642 IsRet ? FunAttrs.getRetAttrs() : FunAttrs.getParamAttrs(OrigIdx);
11643 if (Attrs.hasAttribute(Attribute::SExt) && !Flags.isSExt()) {
11644 errs() << "ERROR: Missing SExt on arg " << OrigIdx << ".\n";
11645 return false;
11646 }
11647 if (Attrs.hasAttribute(Attribute::ZExt) && !Flags.isZExt()) {
11648 errs() << "ERROR: Missing ZExt on arg " << OrigIdx << ".\n";
11649 return false;
11650 }
11651 }
11652 }
11653
11654 return true;
11655}
11656
11658 Module &M, const LibcallLoweringInfo &Libcalls) const {
11659 StringRef GuardMode = M.getStackProtectorGuard();
11660
11661 // In the TLS case, no symbol needs to be inserted.
11662 if (GuardMode == "tls" || GuardMode.empty())
11663 return;
11664
11665 // Otherwise (in the global case), insert the appropriate global variable.
11667}
unsigned NumParts
return SDValue()
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
AMDGPU Register Bank Select
static bool isZeroVector(SDValue N)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
Function Alias Analysis Results
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
#define Check(C,...)
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
static bool isSelectPseudo(MachineInstr &MI)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
static bool isUndef(const MachineInstr &MI)
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
uint64_t High
uint64_t IntrinsicInst * II
#define P(N)
static constexpr MCPhysReg SPReg
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
const char * Msg
This file defines the SmallSet class.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL, Comparison C, SDValue TrueOp, SDValue FalseOp)
static SmallVector< SDValue, 4 > simplifyAssumingCCVal(SDValue &Val, SDValue &CC, SelectionDAG &DAG)
static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static void adjustForLTGFR(Comparison &C)
static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0, SDValue Op1)
#define CONV(X)
static cl::opt< bool > EnableIntArgExtCheck("argext-abi-check", cl::init(false), cl::desc("Verify that narrow int args are properly extended per the " "SystemZ ABI."))
static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG)
static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT, unsigned Opcode, SDValue Op0, SDValue Op1, SDValue &Even, SDValue &Odd)
static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG)
static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Value)
static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In)
static bool isSimpleShift(SDValue N, unsigned &ShiftVal)
static SDValue mergeHighParts(SelectionDAG &DAG, const SDLoc &DL, unsigned MergedBits, EVT VT, SDValue Op0, SDValue Op1)
static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart, SDNode *&HiPart)
static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1)
static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num)
static bool isVectorElementSwap(ArrayRef< int > M, EVT VT)
static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL, SDValue &AlignedAddr, SDValue &BitShift, SDValue &NegBitShift)
static bool isShlDoublePermute(const SmallVectorImpl< int > &Bytes, unsigned &StartIndex, unsigned &OpNo0, unsigned &OpNo1)
static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL, const Permute &P, SDValue Op0, SDValue Op1)
static SDNode * emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op, unsigned Opcode)
static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg)
static void adjustForStackGuardCompare(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode, unsigned &CCValid)
static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend, SDValue Op0, SDValue Op1, SDValue &Hi, SDValue &Lo)
static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart, SDNode *&HiPart)
static void createPHIsForSelects(SmallVector< MachineInstr *, 8 > &Selects, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL, SDValue *Ops, const SmallVectorImpl< int > &Bytes)
static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode, bool &Invert)
static unsigned CCMaskForCondCode(ISD::CondCode CC)
static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static void adjustForFNeg(Comparison &C)
static bool isScalarToVector(SDValue Op)
static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg, unsigned CCValid, unsigned CCMask)
static bool matchPermute(const SmallVectorImpl< int > &Bytes, const Permute &P, unsigned &OpNo0, unsigned &OpNo1)
static bool isAddCarryChain(SDValue Carry)
static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static MachineOperand earlyUseOperand(MachineOperand Op)
static bool canUseSiblingCall(const CCState &ArgCCInfo, SmallVectorImpl< CCValAssign > &ArgLocs, SmallVectorImpl< ISD::OutputArg > &Outs)
static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA, SDLoc &DL, SDValue &Chain)
static SDValue convertToF16(SDValue Op, SelectionDAG &DAG)
static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask, SelectionDAG &DAG)
static bool shouldSwapCmpOperands(const Comparison &C)
static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType)
static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL, unsigned Offset, bool LoadAdr=false)
static SDNode * emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op, unsigned Opcode)
static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static bool getVPermMask(SDValue ShuffleOp, SmallVectorImpl< int > &Bytes)
static const Permute PermuteForms[]
static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart, SDValue &HiPart)
static std::pair< SDValue, int > findCCUse(const SDValue &Val, unsigned Depth=0)
static bool isSubBorrowChain(SDValue Carry)
static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static bool analyzeArgSplit(const SmallVectorImpl< ArgTy > &Args, SmallVector< CCValAssign, 16 > &ArgLocs, unsigned I, MVT &PartVT, unsigned &NumParts)
static bool isInBoundsConstantVectorIndex(EVT VecVT, SDValue Idx)
static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts, unsigned OpNo)
static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op, bool IsNegative)
static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth, unsigned OpNo)
static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src, const SDLoc &SL)
static SDValue tryBuildVectorShuffle(SelectionDAG &DAG, BuildVectorSDNode *BVN)
static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG)
static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode)
static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In)
static SDValue MergeInputChains(SDNode *N1, SDNode *N2)
static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src, const SDLoc &SL)
static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, uint64_t Mask, uint64_t CmpVal, unsigned ICmpType)
static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid)
static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL, SDValue Op, SDValue Chain)
static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, ISD::CondCode Cond, const SDLoc &DL, SDValue Chain=SDValue(), bool IsSignaling=false)
static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB)
static Register forceReg(MachineInstr &MI, MachineOperand &Base, const SystemZInstrInfo *TII)
static bool is32Bit(EVT VT)
static std::pair< unsigned, const TargetRegisterClass * > parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC, const unsigned *Map, unsigned Size)
static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG, const SystemZSubtarget &Subtarget, SDValue &Op)
static bool matchDoublePermute(const SmallVectorImpl< int > &Bytes, const Permute &P, SmallVectorImpl< int > &Transform)
static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode, SDValue Call, unsigned CCValid, uint64_t CC, ISD::CondCode Cond)
static Register getOrCreateADAVirtualRegister(MachineFunction &MF)
static SDValue buildFPVecFromScalars4(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SmallVectorImpl< SDValue > &Elems, unsigned Pos)
static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg)
static AddressingMode getLoadStoreAddrMode(bool HasVector, Type *Ty)
static void printFunctionArgExts(const Function *F, raw_ostream &OS)
static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op0, SDValue Op1)
static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth, unsigned OpNo)
static bool getShuffleInput(const SmallVectorImpl< int > &Bytes, unsigned Start, unsigned BytesPerElement, int &Base)
static AddressingMode supportedAddressingMode(Instruction *I, bool HasVector)
static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart, SDValue &HiPart)
static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
Value * RHS
Value * LHS
BinaryOperator * Mul
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition APInt.h:1350
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition APInt.h:254
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
bool isSingleWord() const
Determine if this APInt just has one word to store value.
Definition APInt.h:318
LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition APInt.cpp:393
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition APInt.h:860
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
an instruction that atomically reads a memory location, combines it with another value,...
@ Add
*p = old + v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
BinOp getOperation() const
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:448
bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
Definition Attributes.h:532
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
static LLVM_ABI StringRef getNameFromAttrKind(Attribute::AttrKind AttrKind)
LLVM Basic Block Representation.
Definition BasicBlock.h:62
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
LLVM_ABI bool isConstant() const
CCState - This class holds information needed while lowering arguments and return values.
LLVM_ABI void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
LLVM_ABI bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
LLVM_ABI void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
LLVM_ABI void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
bool needsCustom() const
bool isExtInLoc() const
int64_t getLocMemOffset() const
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
MachineConstantPoolValue * getMachineCPVal() const
const Constant * getConstVal() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
LLVM_ABI unsigned getPointerSize(unsigned AS=0) const
The pointer representation size in bytes, rounded up to a whole number of bytes.
A debug info location.
Definition DebugLoc.h:126
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
bool hasAddressTaken(const User **=nullptr, bool IgnoreCallbackUses=false, bool IgnoreAssumeLikeCalls=true, bool IngoreLLVMUsed=false, bool IgnoreARCAttachedCall=false, bool IgnoreCastedDirectCall=false) const
hasAddressTaken - returns true if there are any uses of this function other than direct calls or invo...
Definition Function.cpp:941
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:766
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
Definition Function.cpp:778
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:329
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
LLVM_ABI const GlobalObject * getAliaseeObject() const
Definition Globals.cpp:730
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
bool hasInternalLinkage() const
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
Tracks which library functions to use for a particular subtarget or function.
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
static auto integer_fixedlen_vector_valuetypes()
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isVector() const
Return true if this is a vector value type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setMaxCallFrameSize(uint64_t S)
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setFrameAddressIsTaken(bool T)
uint64_t getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
reverse_iterator rbegin()
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
bool killsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr kills the specified register.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
void setFlags(SDNodeFlags NewFlags)
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS, const SDNodeFlags Flags=SDNodeFlags())
Return an AddrSpaceCastSDNode.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
SDValue getGLOBAL_OFFSET_TABLE(EVT VT)
Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
const_iterator begin() const
Definition SmallSet.h:216
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
size_type size() const
Definition SmallSet.h:171
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void resize(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition StringRef.h:490
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
Definition StringRef.h:720
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
iterator end() const
Definition StringRef.h:116
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
A SystemZ-specific class detailing special use registers particular for calling conventions.
static SystemZConstantPoolValue * Create(const GlobalValue *GV, SystemZCP::SystemZCPModifier Modifier)
SystemZCallingConventionRegisters * getSpecialRegisters() const
AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool hasInlineStackProbe(const MachineFunction &MF) const override
Returns true if stack probing through inline assembly is requested.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
MachineBasicBlock * emitEHSjLjSetJmp(MachineInstr &MI, MachineBasicBlock *MBB) const
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &, EVT) const override
Return the ValueType of the result of SETCC operations.
bool allowTruncateForTailCall(Type *, Type *) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, const SDLoc &DL, const AsmOperandInfo &Constraint, SelectionDAG &DAG) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
MachineBasicBlock * emitEHSjLjLongJmp(MachineInstr &MI, MachineBasicBlock *MBB) const
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
std::pair< SDValue, SDValue > makeExternalCall(SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT, ArrayRef< SDValue > Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL, bool DoesNotReturn, bool IsReturnValueUsed) const
void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const override
Insert SSP declaration if global stack protector is used.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, unsigned Depth) const override
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
SystemZTargetLowering(const TargetMachine &TM, const SystemZSubtarget &STI)
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
FPImmAction getFPImmAction(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns how the legalizer should materialize the specified FP immediate.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool isTypeDesirableForOp(SDNode *N, EVT VT) const override
Overload that takes the specific node being optimized.
TargetLowering::ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
Determine if the target supports unaligned memory accesses.
const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const override
Returns a 0 terminated array of registers that can be safely used as scratch registers.
TargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isTruncateFree(Type *, Type *) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
SDValue useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, MVT VT, SDValue Arg, SDLoc DL, SDValue Chain, bool IsStrict) const
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine the number of bits in the operation that are sign bits.
void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, const Function *F) const override
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool findOptimalMemOpLowering(LLVMContext &Context, std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, EVT *LargestVT=nullptr) const override
Determines the optimal series of memory ops to replace the memset / memcpy.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override
AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
unsigned getStackProbeSize(const MachineFunction &MF) const
XPLINK64 calling convention specific use registers Particular to z/OS when in 64 bit mode.
Information about stack frame layout on the target.
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual void insertSSPDeclarations(Module &M, const LibcallLoweringInfo &Libcalls) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Let target indicate that an extending atomic load of the specified type is legal.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
virtual const TargetRegisterClass * getRepRegClassFor(MVT VT) const
Return the 'representative' register class for the specified value type.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool findOptimalMemOpLowering(LLVMContext &Context, std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, EVT *LargestVT=nullptr) const
Determines the optimal series of memory ops to replace the memset / memcpy.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
TargetLowering(const TargetLowering &)=delete
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned VerifyArgABICompliance
When set to true, call/return argument extensions of narrow integers are verified in the target backe...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
User * getUser() const
Returns the User that contains this Use.
Definition Use.h:61
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
user_iterator user_begin()
Definition Value.h:404
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
Definition CallingConv.h:50
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
LLVM_ABI bool isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 ...
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition ISDOpcodes.h:170
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ PSEUDO_FMIN
PSEUDO_FMIN is strictly equivalent to op0 olt op1 ?
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ GlobalAddress
Definition ISDOpcodes.h:90
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ STRICT_FMINIMUM
Definition ISDOpcodes.h:476
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition ISDOpcodes.h:942
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:282
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ STRICT_PSEUDO_FMAX
Definition ISDOpcodes.h:465
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition ISDOpcodes.h:441
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ GlobalTLSAddress
Definition ISDOpcodes.h:91
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ STRICT_UINT_TO_FP
Definition ISDOpcodes.h:490
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ STRICT_PSEUDO_FMIN
Definition ISDOpcodes.h:464
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ BR_JT
BR_JT - Jumptable branch.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ STRICT_FMAXIMUM
Definition ISDOpcodes.h:475
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition ISDOpcodes.h:931
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition ISDOpcodes.h:489
@ STRICT_FROUNDEVEN
Definition ISDOpcodes.h:469
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:112
@ STRICT_FP_TO_UINT
Definition ISDOpcodes.h:483
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:482
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition ISDOpcodes.h:510
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition ISDOpcodes.h:430
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition ISDOpcodes.h:953
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ STRICT_FNEARBYINT
Definition ISDOpcodes.h:461
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition ISDOpcodes.h:164
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
auto m_Cmp()
Matches any compare instruction and ignore it.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
auto m_Value()
Match an arbitrary value and ignore it.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ System
Synchronized with respect to all concurrently executing threads.
Definition LLVMContext.h:66
const unsigned GR64Regs[16]
const unsigned VR128Regs[32]
const unsigned VR16Regs[32]
const unsigned GR128Regs[16]
const unsigned FP32Regs[16]
const unsigned FP16Regs[16]
const unsigned GR32Regs[16]
const unsigned FP64Regs[16]
const int64_t ELFCallFrameSize
const unsigned VR64Regs[32]
const unsigned FP128Regs[16]
const unsigned VR32Regs[32]
unsigned odd128(bool Is32bit)
const unsigned CCMASK_CMP_GE
Definition SystemZ.h:46
static bool isImmHH(uint64_t Val)
Definition SystemZ.h:182
const unsigned CCMASK_TEND
Definition SystemZ.h:103
const unsigned CCMASK_CS_EQ
Definition SystemZ.h:73
const unsigned CCMASK_TBEGIN
Definition SystemZ.h:98
const unsigned CCMASK_0
Definition SystemZ.h:33
const MCPhysReg ELFArgFPRs[ELFNumArgFPRs]
MachineBasicBlock * splitBlockBefore(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB)
const unsigned CCMASK_TM_SOME_1
Definition SystemZ.h:88
const unsigned CCMASK_LOGICAL_CARRY
Definition SystemZ.h:66
const unsigned TDCMASK_NORMAL_MINUS
Definition SystemZ.h:128
const unsigned CCMASK_TDC
Definition SystemZ.h:115
const unsigned CCMASK_FCMP
Definition SystemZ.h:54
const unsigned CCMASK_TM_SOME_0
Definition SystemZ.h:87
static bool isImmHL(uint64_t Val)
Definition SystemZ.h:177
const unsigned TDCMASK_SUBNORMAL_MINUS
Definition SystemZ.h:130
const unsigned PFD_READ
Definition SystemZ.h:121
const unsigned CCMASK_1
Definition SystemZ.h:34
const unsigned TDCMASK_NORMAL_PLUS
Definition SystemZ.h:127
const unsigned PFD_WRITE
Definition SystemZ.h:122
const unsigned CCMASK_CMP_GT
Definition SystemZ.h:43
const unsigned TDCMASK_QNAN_MINUS
Definition SystemZ.h:134
const unsigned CCMASK_CS
Definition SystemZ.h:75
const unsigned CCMASK_ANY
Definition SystemZ.h:37
const unsigned CCMASK_ARITH
Definition SystemZ.h:61
const unsigned CCMASK_TM_MIXED_MSB_0
Definition SystemZ.h:84
const unsigned TDCMASK_SUBNORMAL_PLUS
Definition SystemZ.h:129
static bool isImmLL(uint64_t Val)
Definition SystemZ.h:167
const unsigned VectorBits
Definition SystemZ.h:160
static bool isImmLH(uint64_t Val)
Definition SystemZ.h:172
MachineBasicBlock * emitBlockAfter(MachineBasicBlock *MBB)
const unsigned TDCMASK_INFINITY_PLUS
Definition SystemZ.h:131
unsigned reverseCCMask(unsigned CCMask)
const unsigned CCMASK_TM_ALL_0
Definition SystemZ.h:83
const unsigned IPM_CC
Definition SystemZ.h:118
const unsigned CCMASK_CMP_LE
Definition SystemZ.h:45
const unsigned CCMASK_CMP_O
Definition SystemZ.h:50
const unsigned CCMASK_CMP_EQ
Definition SystemZ.h:41
const unsigned VectorBytes
Definition SystemZ.h:164
const unsigned TDCMASK_INFINITY_MINUS
Definition SystemZ.h:132
const unsigned CCMASK_ICMP
Definition SystemZ.h:53
const unsigned CCMASK_VCMP_ALL
Definition SystemZ.h:107
const unsigned CCMASK_VCMP_NONE
Definition SystemZ.h:109
MachineBasicBlock * splitBlockAfter(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB)
const unsigned CCMASK_VCMP
Definition SystemZ.h:110
const unsigned CCMASK_TM_MIXED_MSB_1
Definition SystemZ.h:85
const unsigned CCMASK_TM_MSB_0
Definition SystemZ.h:89
const unsigned CCMASK_ARITH_OVERFLOW
Definition SystemZ.h:60
const unsigned CCMASK_CS_NE
Definition SystemZ.h:74
const unsigned TDCMASK_SNAN_PLUS
Definition SystemZ.h:135
const unsigned CCMASK_TM
Definition SystemZ.h:91
const unsigned CCMASK_3
Definition SystemZ.h:36
const unsigned CCMASK_NONE
Definition SystemZ.h:32
const unsigned CCMASK_CMP_LT
Definition SystemZ.h:42
const unsigned CCMASK_CMP_NE
Definition SystemZ.h:44
const unsigned TDCMASK_ZERO_PLUS
Definition SystemZ.h:125
const unsigned TDCMASK_QNAN_PLUS
Definition SystemZ.h:133
const unsigned TDCMASK_ZERO_MINUS
Definition SystemZ.h:126
unsigned even128(bool Is32bit)
const unsigned CCMASK_TM_ALL_1
Definition SystemZ.h:86
const unsigned CCMASK_LOGICAL_BORROW
Definition SystemZ.h:68
const unsigned ELFNumArgFPRs
const unsigned CCMASK_CMP_UO
Definition SystemZ.h:49
const unsigned CCMASK_LOGICAL
Definition SystemZ.h:70
const unsigned CCMASK_TM_MSB_1
Definition SystemZ.h:90
const unsigned TDCMASK_SNAN_MINUS
Definition SystemZ.h:136
initializer< Ty > init(const Ty &Val)
support::ulittle32_t Word
Definition IRSymtab.h:53
@ User
could "use" a pointer
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
NodeAddr< CodeNode * > Code
Definition RDFGraph.h:388
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Define
Register definition.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Done
Definition Threading.h:60
@ Load
The value being inserted comes from a load (InsertElement only).
testing::Matcher< const detail::ErrorHolder & > Failed()
Definition Error.h:198
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
constexpr T maskLeadingOnes(unsigned N)
Create a bitmask with the N left-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:89
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
LLVM_ABI void dumpBytes(ArrayRef< uint8_t > Bytes, raw_ostream &OS)
Convert ‘Bytes’ to a hex string and output to ‘OS’.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition bit.h:362
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
@ Success
The lock was released successfully.
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
@ BeforeLegalizeTypes
Definition DAGCombine.h:16
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
ExceptionHandling
Definition CodeGen.h:54
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Definition UndefPoison.h:20
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition MathExtras.h:572
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
#define EQ(a, b)
Definition regexec.c:65
AddressingMode(bool LongDispl, bool IdxReg)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition ValueTypes.h:90
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
Definition ValueTypes.h:286
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
bool isVectorOf(EVT EltVT) const
Return true if this is a vector with matching element type.
Definition ValueTypes.h:196
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition KnownBits.h:325
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:146
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.