Bug Summary

File:lib/Target/SystemZ/SystemZISelLowering.cpp
Warning:line 4409, column 41
The result of the left shift is undefined due to shifting by '18446744073709551615', which is greater or equal to the width of type 'uint64_t'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SystemZISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn326551/build-llvm/lib/Target/SystemZ -I /build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ -I /build/llvm-toolchain-snapshot-7~svn326551/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn326551/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn326551/build-llvm/lib/Target/SystemZ -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-03-02-155150-1477-1 -x c++ /build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SystemZTargetLowering class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SystemZISelLowering.h"
15#include "SystemZCallingConv.h"
16#include "SystemZConstantPoolValue.h"
17#include "SystemZMachineFunctionInfo.h"
18#include "SystemZTargetMachine.h"
19#include "llvm/CodeGen/CallingConvLower.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/KnownBits.h"
27#include <cctype>
28
29using namespace llvm;
30
31#define DEBUG_TYPE"systemz-lower" "systemz-lower"
32
33namespace {
34// Represents information about a comparison.
35struct Comparison {
36 Comparison(SDValue Op0In, SDValue Op1In)
37 : Op0(Op0In), Op1(Op1In), Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
38
39 // The operands to the comparison.
40 SDValue Op0, Op1;
41
42 // The opcode that should be used to compare Op0 and Op1.
43 unsigned Opcode;
44
45 // A SystemZICMP value. Only used for integer comparisons.
46 unsigned ICmpType;
47
48 // The mask of CC values that Opcode can produce.
49 unsigned CCValid;
50
51 // The mask of CC values for which the original condition is true.
52 unsigned CCMask;
53};
54} // end anonymous namespace
55
56// Classify VT as either 32 or 64 bit.
57static bool is32Bit(EVT VT) {
58 switch (VT.getSimpleVT().SimpleTy) {
59 case MVT::i32:
60 return true;
61 case MVT::i64:
62 return false;
63 default:
64 llvm_unreachable("Unsupported type")::llvm::llvm_unreachable_internal("Unsupported type", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 64)
;
65 }
66}
67
68// Return a version of MachineOperand that can be safely used before the
69// final use.
70static MachineOperand earlyUseOperand(MachineOperand Op) {
71 if (Op.isReg())
72 Op.setIsKill(false);
73 return Op;
74}
75
76SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
77 const SystemZSubtarget &STI)
78 : TargetLowering(TM), Subtarget(STI) {
79 MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize());
80
81 // Set up the register classes.
82 if (Subtarget.hasHighWord())
83 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass);
84 else
85 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass);
86 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
87 if (Subtarget.hasVector()) {
88 addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
89 addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
90 } else {
91 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
92 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
93 }
94 if (Subtarget.hasVectorEnhancements1())
95 addRegisterClass(MVT::f128, &SystemZ::VR128BitRegClass);
96 else
97 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass);
98
99 if (Subtarget.hasVector()) {
100 addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass);
101 addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass);
102 addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass);
103 addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass);
104 addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass);
105 addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass);
106 }
107
108 // Compute derived properties from the register classes
109 computeRegisterProperties(Subtarget.getRegisterInfo());
110
111 // Set up special registers.
112 setStackPointerRegisterToSaveRestore(SystemZ::R15D);
113
114 // TODO: It may be better to default to latency-oriented scheduling, however
115 // LLVM's current latency-oriented scheduler can't handle physreg definitions
116 // such as SystemZ has with CC, so set this to the register-pressure
117 // scheduler, because it can.
118 setSchedulingPreference(Sched::RegPressure);
119
120 setBooleanContents(ZeroOrOneBooleanContent);
121 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
122
123 // Instructions are strings of 2-byte aligned 2-byte values.
124 setMinFunctionAlignment(2);
125
126 // Handle operations that are handled in a similar way for all types.
127 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
128 I <= MVT::LAST_FP_VALUETYPE;
129 ++I) {
130 MVT VT = MVT::SimpleValueType(I);
131 if (isTypeLegal(VT)) {
132 // Lower SET_CC into an IPM-based sequence.
133 setOperationAction(ISD::SETCC, VT, Custom);
134
135 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
136 setOperationAction(ISD::SELECT, VT, Expand);
137
138 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
139 setOperationAction(ISD::SELECT_CC, VT, Custom);
140 setOperationAction(ISD::BR_CC, VT, Custom);
141 }
142 }
143
144 // Expand jump table branches as address arithmetic followed by an
145 // indirect jump.
146 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
147
148 // Expand BRCOND into a BR_CC (see above).
149 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
150
151 // Handle integer types.
152 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
153 I <= MVT::LAST_INTEGER_VALUETYPE;
154 ++I) {
155 MVT VT = MVT::SimpleValueType(I);
156 if (isTypeLegal(VT)) {
157 // Expand individual DIV and REMs into DIVREMs.
158 setOperationAction(ISD::SDIV, VT, Expand);
159 setOperationAction(ISD::UDIV, VT, Expand);
160 setOperationAction(ISD::SREM, VT, Expand);
161 setOperationAction(ISD::UREM, VT, Expand);
162 setOperationAction(ISD::SDIVREM, VT, Custom);
163 setOperationAction(ISD::UDIVREM, VT, Custom);
164
165 // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and
166 // stores, putting a serialization instruction after the stores.
167 setOperationAction(ISD::ATOMIC_LOAD, VT, Custom);
168 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
169
170 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
171 // available, or if the operand is constant.
172 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
173
174 // Use POPCNT on z196 and above.
175 if (Subtarget.hasPopulationCount())
176 setOperationAction(ISD::CTPOP, VT, Custom);
177 else
178 setOperationAction(ISD::CTPOP, VT, Expand);
179
180 // No special instructions for these.
181 setOperationAction(ISD::CTTZ, VT, Expand);
182 setOperationAction(ISD::ROTR, VT, Expand);
183
184 // Use *MUL_LOHI where possible instead of MULH*.
185 setOperationAction(ISD::MULHS, VT, Expand);
186 setOperationAction(ISD::MULHU, VT, Expand);
187 setOperationAction(ISD::SMUL_LOHI, VT, Custom);
188 setOperationAction(ISD::UMUL_LOHI, VT, Custom);
189
190 // Only z196 and above have native support for conversions to unsigned.
191 // On z10, promoting to i64 doesn't generate an inexact condition for
192 // values that are outside the i32 range but in the i64 range, so use
193 // the default expansion.
194 if (!Subtarget.hasFPExtension())
195 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
196 }
197 }
198
199 // Type legalization will convert 8- and 16-bit atomic operations into
200 // forms that operate on i32s (but still keeping the original memory VT).
201 // Lower them into full i32 operations.
202 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom);
203 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom);
204 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
205 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
206 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom);
207 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom);
208 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom);
209 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom);
210 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom);
211 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom);
212 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom);
213
214 // Even though i128 is not a legal type, we still need to custom lower
215 // the atomic operations in order to exploit SystemZ instructions.
216 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
217 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
218
219 // We can use the CC result of compare-and-swap to implement
220 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
221 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Custom);
222 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Custom);
223 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
224
225 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
226
227 // Traps are legal, as we will convert them to "j .+2".
228 setOperationAction(ISD::TRAP, MVT::Other, Legal);
229
230 // z10 has instructions for signed but not unsigned FP conversion.
231 // Handle unsigned 32-bit types as signed 64-bit types.
232 if (!Subtarget.hasFPExtension()) {
233 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote);
234 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
235 }
236
237 // We have native support for a 64-bit CTLZ, via FLOGR.
238 setOperationAction(ISD::CTLZ, MVT::i32, Promote);
239 setOperationAction(ISD::CTLZ, MVT::i64, Legal);
240
241 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
242 setOperationAction(ISD::OR, MVT::i64, Custom);
243
244 // FIXME: Can we support these natively?
245 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
246 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
247 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
248
249 // We have native instructions for i8, i16 and i32 extensions, but not i1.
250 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
251 for (MVT VT : MVT::integer_valuetypes()) {
252 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
253 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
254 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
255 }
256
257 // Handle the various types of symbolic address.
258 setOperationAction(ISD::ConstantPool, PtrVT, Custom);
259 setOperationAction(ISD::GlobalAddress, PtrVT, Custom);
260 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom);
261 setOperationAction(ISD::BlockAddress, PtrVT, Custom);
262 setOperationAction(ISD::JumpTable, PtrVT, Custom);
263
264 // We need to handle dynamic allocations specially because of the
265 // 160-byte area at the bottom of the stack.
266 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
267 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, PtrVT, Custom);
268
269 // Use custom expanders so that we can force the function to use
270 // a frame pointer.
271 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom);
272 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom);
273
274 // Handle prefetches with PFD or PFDRL.
275 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
276
277 for (MVT VT : MVT::vector_valuetypes()) {
278 // Assume by default that all vector operations need to be expanded.
279 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
280 if (getOperationAction(Opcode, VT) == Legal)
281 setOperationAction(Opcode, VT, Expand);
282
283 // Likewise all truncating stores and extending loads.
284 for (MVT InnerVT : MVT::vector_valuetypes()) {
285 setTruncStoreAction(VT, InnerVT, Expand);
286 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
287 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
288 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
289 }
290
291 if (isTypeLegal(VT)) {
292 // These operations are legal for anything that can be stored in a
293 // vector register, even if there is no native support for the format
294 // as such. In particular, we can do these for v4f32 even though there
295 // are no specific instructions for that format.
296 setOperationAction(ISD::LOAD, VT, Legal);
297 setOperationAction(ISD::STORE, VT, Legal);
298 setOperationAction(ISD::VSELECT, VT, Legal);
299 setOperationAction(ISD::BITCAST, VT, Legal);
300 setOperationAction(ISD::UNDEF, VT, Legal);
301
302 // Likewise, except that we need to replace the nodes with something
303 // more specific.
304 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
305 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
306 }
307 }
308
309 // Handle integer vector types.
310 for (MVT VT : MVT::integer_vector_valuetypes()) {
311 if (isTypeLegal(VT)) {
312 // These operations have direct equivalents.
313 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal);
314 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Legal);
315 setOperationAction(ISD::ADD, VT, Legal);
316 setOperationAction(ISD::SUB, VT, Legal);
317 if (VT != MVT::v2i64)
318 setOperationAction(ISD::MUL, VT, Legal);
319 setOperationAction(ISD::AND, VT, Legal);
320 setOperationAction(ISD::OR, VT, Legal);
321 setOperationAction(ISD::XOR, VT, Legal);
322 if (Subtarget.hasVectorEnhancements1())
323 setOperationAction(ISD::CTPOP, VT, Legal);
324 else
325 setOperationAction(ISD::CTPOP, VT, Custom);
326 setOperationAction(ISD::CTTZ, VT, Legal);
327 setOperationAction(ISD::CTLZ, VT, Legal);
328
329 // Convert a GPR scalar to a vector by inserting it into element 0.
330 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
331
332 // Use a series of unpacks for extensions.
333 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
334 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
335
336 // Detect shifts by a scalar amount and convert them into
337 // V*_BY_SCALAR.
338 setOperationAction(ISD::SHL, VT, Custom);
339 setOperationAction(ISD::SRA, VT, Custom);
340 setOperationAction(ISD::SRL, VT, Custom);
341
342 // At present ROTL isn't matched by DAGCombiner. ROTR should be
343 // converted into ROTL.
344 setOperationAction(ISD::ROTL, VT, Expand);
345 setOperationAction(ISD::ROTR, VT, Expand);
346
347 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
348 // and inverting the result as necessary.
349 setOperationAction(ISD::SETCC, VT, Custom);
350 }
351 }
352
353 if (Subtarget.hasVector()) {
354 // There should be no need to check for float types other than v2f64
355 // since <2 x f32> isn't a legal type.
356 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
357 setOperationAction(ISD::FP_TO_SINT, MVT::v2f64, Legal);
358 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
359 setOperationAction(ISD::FP_TO_UINT, MVT::v2f64, Legal);
360 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
361 setOperationAction(ISD::SINT_TO_FP, MVT::v2f64, Legal);
362 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
363 setOperationAction(ISD::UINT_TO_FP, MVT::v2f64, Legal);
364 }
365
366 // Handle floating-point types.
367 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
368 I <= MVT::LAST_FP_VALUETYPE;
369 ++I) {
370 MVT VT = MVT::SimpleValueType(I);
371 if (isTypeLegal(VT)) {
372 // We can use FI for FRINT.
373 setOperationAction(ISD::FRINT, VT, Legal);
374
375 // We can use the extended form of FI for other rounding operations.
376 if (Subtarget.hasFPExtension()) {
377 setOperationAction(ISD::FNEARBYINT, VT, Legal);
378 setOperationAction(ISD::FFLOOR, VT, Legal);
379 setOperationAction(ISD::FCEIL, VT, Legal);
380 setOperationAction(ISD::FTRUNC, VT, Legal);
381 setOperationAction(ISD::FROUND, VT, Legal);
382 }
383
384 // No special instructions for these.
385 setOperationAction(ISD::FSIN, VT, Expand);
386 setOperationAction(ISD::FCOS, VT, Expand);
387 setOperationAction(ISD::FSINCOS, VT, Expand);
388 setOperationAction(ISD::FREM, VT, Expand);
389 setOperationAction(ISD::FPOW, VT, Expand);
390 }
391 }
392
393 // Handle floating-point vector types.
394 if (Subtarget.hasVector()) {
395 // Scalar-to-vector conversion is just a subreg.
396 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
397 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
398
399 // Some insertions and extractions can be done directly but others
400 // need to go via integers.
401 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
402 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
403 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
404 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
405
406 // These operations have direct equivalents.
407 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
408 setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
409 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
410 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
411 setOperationAction(ISD::FMA, MVT::v2f64, Legal);
412 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
413 setOperationAction(ISD::FABS, MVT::v2f64, Legal);
414 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
415 setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
416 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
417 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
418 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
419 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
420 setOperationAction(ISD::FROUND, MVT::v2f64, Legal);
421 }
422
423 // The vector enhancements facility 1 has instructions for these.
424 if (Subtarget.hasVectorEnhancements1()) {
425 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
426 setOperationAction(ISD::FNEG, MVT::v4f32, Legal);
427 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
428 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
429 setOperationAction(ISD::FMA, MVT::v4f32, Legal);
430 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
431 setOperationAction(ISD::FABS, MVT::v4f32, Legal);
432 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
433 setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
434 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
435 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
436 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
437 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
438 setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
439
440 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
441 setOperationAction(ISD::FMAXNAN, MVT::f64, Legal);
442 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
443 setOperationAction(ISD::FMINNAN, MVT::f64, Legal);
444
445 setOperationAction(ISD::FMAXNUM, MVT::v2f64, Legal);
446 setOperationAction(ISD::FMAXNAN, MVT::v2f64, Legal);
447 setOperationAction(ISD::FMINNUM, MVT::v2f64, Legal);
448 setOperationAction(ISD::FMINNAN, MVT::v2f64, Legal);
449
450 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
451 setOperationAction(ISD::FMAXNAN, MVT::f32, Legal);
452 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
453 setOperationAction(ISD::FMINNAN, MVT::f32, Legal);
454
455 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
456 setOperationAction(ISD::FMAXNAN, MVT::v4f32, Legal);
457 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
458 setOperationAction(ISD::FMINNAN, MVT::v4f32, Legal);
459
460 setOperationAction(ISD::FMAXNUM, MVT::f128, Legal);
461 setOperationAction(ISD::FMAXNAN, MVT::f128, Legal);
462 setOperationAction(ISD::FMINNUM, MVT::f128, Legal);
463 setOperationAction(ISD::FMINNAN, MVT::f128, Legal);
464 }
465
466 // We have fused multiply-addition for f32 and f64 but not f128.
467 setOperationAction(ISD::FMA, MVT::f32, Legal);
468 setOperationAction(ISD::FMA, MVT::f64, Legal);
469 if (Subtarget.hasVectorEnhancements1())
470 setOperationAction(ISD::FMA, MVT::f128, Legal);
471 else
472 setOperationAction(ISD::FMA, MVT::f128, Expand);
473
474 // We don't have a copysign instruction on vector registers.
475 if (Subtarget.hasVectorEnhancements1())
476 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
477
478 // Needed so that we don't try to implement f128 constant loads using
479 // a load-and-extend of a f80 constant (in cases where the constant
480 // would fit in an f80).
481 for (MVT VT : MVT::fp_valuetypes())
482 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
483
484 // We don't have extending load instruction on vector registers.
485 if (Subtarget.hasVectorEnhancements1()) {
486 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
487 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
488 }
489
490 // Floating-point truncation and stores need to be done separately.
491 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
492 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
493 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
494
495 // We have 64-bit FPR<->GPR moves, but need special handling for
496 // 32-bit forms.
497 if (!Subtarget.hasVector()) {
498 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
499 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
500 }
501
502 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
503 // structure, but VAEND is a no-op.
504 setOperationAction(ISD::VASTART, MVT::Other, Custom);
505 setOperationAction(ISD::VACOPY, MVT::Other, Custom);
506 setOperationAction(ISD::VAEND, MVT::Other, Expand);
507
508 // Codes for which we want to perform some z-specific combinations.
509 setTargetDAGCombine(ISD::ZERO_EXTEND);
510 setTargetDAGCombine(ISD::SIGN_EXTEND);
511 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
512 setTargetDAGCombine(ISD::STORE);
513 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
514 setTargetDAGCombine(ISD::FP_ROUND);
515 setTargetDAGCombine(ISD::BSWAP);
516 setTargetDAGCombine(ISD::SHL);
517 setTargetDAGCombine(ISD::SRA);
518 setTargetDAGCombine(ISD::SRL);
519 setTargetDAGCombine(ISD::ROTL);
520
521 // Handle intrinsics.
522 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
523 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
524
525 // We want to use MVC in preference to even a single load/store pair.
526 MaxStoresPerMemcpy = 0;
527 MaxStoresPerMemcpyOptSize = 0;
528
529 // The main memset sequence is a byte store followed by an MVC.
530 // Two STC or MV..I stores win over that, but the kind of fused stores
531 // generated by target-independent code don't when the byte value is
532 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
533 // than "STC;MVC". Handle the choice in target-specific code instead.
534 MaxStoresPerMemset = 0;
535 MaxStoresPerMemsetOptSize = 0;
536}
537
538EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
539 LLVMContext &, EVT VT) const {
540 if (!VT.isVector())
541 return MVT::i32;
542 return VT.changeVectorElementTypeToInteger();
543}
544
545bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
546 VT = VT.getScalarType();
547
548 if (!VT.isSimple())
549 return false;
550
551 switch (VT.getSimpleVT().SimpleTy) {
552 case MVT::f32:
553 case MVT::f64:
554 return true;
555 case MVT::f128:
556 return Subtarget.hasVectorEnhancements1();
557 default:
558 break;
559 }
560
561 return false;
562}
563
564bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
565 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
566 return Imm.isZero() || Imm.isNegZero();
567}
568
569bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
570 // We can use CGFI or CLGFI.
571 return isInt<32>(Imm) || isUInt<32>(Imm);
572}
573
574bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
575 // We can use ALGFI or SLGFI.
576 return isUInt<32>(Imm) || isUInt<32>(-Imm);
577}
578
579bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
580 unsigned,
581 unsigned,
582 bool *Fast) const {
583 // Unaligned accesses should never be slower than the expanded version.
584 // We check specifically for aligned accesses in the few cases where
585 // they are required.
586 if (Fast)
587 *Fast = true;
588 return true;
589}
590
591// Information about the addressing mode for a memory access.
592struct AddressingMode {
593 // True if a long displacement is supported.
594 bool LongDisplacement;
595
596 // True if use of index register is supported.
597 bool IndexReg;
598
599 AddressingMode(bool LongDispl, bool IdxReg) :
600 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
601};
602
603// Return the desired addressing mode for a Load which has only one use (in
604// the same block) which is a Store.
605static AddressingMode getLoadStoreAddrMode(bool HasVector,
606 Type *Ty) {
607 // With vector support a Load->Store combination may be combined to either
608 // an MVC or vector operations and it seems to work best to allow the
609 // vector addressing mode.
610 if (HasVector)
611 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
612
613 // Otherwise only the MVC case is special.
614 bool MVC = Ty->isIntegerTy(8);
615 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
616}
617
618// Return the addressing mode which seems most desirable given an LLVM
619// Instruction pointer.
620static AddressingMode
621supportedAddressingMode(Instruction *I, bool HasVector) {
622 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
623 switch (II->getIntrinsicID()) {
624 default: break;
625 case Intrinsic::memset:
626 case Intrinsic::memmove:
627 case Intrinsic::memcpy:
628 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
629 }
630 }
631
632 if (isa<LoadInst>(I) && I->hasOneUse()) {
633 auto *SingleUser = dyn_cast<Instruction>(*I->user_begin());
634 if (SingleUser->getParent() == I->getParent()) {
635 if (isa<ICmpInst>(SingleUser)) {
636 if (auto *C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
637 if (C->getBitWidth() <= 64 &&
638 (isInt<16>(C->getSExtValue()) || isUInt<16>(C->getZExtValue())))
639 // Comparison of memory with 16 bit signed / unsigned immediate
640 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
641 } else if (isa<StoreInst>(SingleUser))
642 // Load->Store
643 return getLoadStoreAddrMode(HasVector, I->getType());
644 }
645 } else if (auto *StoreI = dyn_cast<StoreInst>(I)) {
646 if (auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
647 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
648 // Load->Store
649 return getLoadStoreAddrMode(HasVector, LoadI->getType());
650 }
651
652 if (HasVector && (isa<LoadInst>(I) || isa<StoreInst>(I))) {
653
654 // * Use LDE instead of LE/LEY for z13 to avoid partial register
655 // dependencies (LDE only supports small offsets).
656 // * Utilize the vector registers to hold floating point
657 // values (vector load / store instructions only support small
658 // offsets).
659
660 Type *MemAccessTy = (isa<LoadInst>(I) ? I->getType() :
661 I->getOperand(0)->getType());
662 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
663 bool IsVectorAccess = MemAccessTy->isVectorTy();
664
665 // A store of an extracted vector element will be combined into a VSTE type
666 // instruction.
667 if (!IsVectorAccess && isa<StoreInst>(I)) {
668 Value *DataOp = I->getOperand(0);
669 if (isa<ExtractElementInst>(DataOp))
670 IsVectorAccess = true;
671 }
672
673 // A load which gets inserted into a vector element will be combined into a
674 // VLE type instruction.
675 if (!IsVectorAccess && isa<LoadInst>(I) && I->hasOneUse()) {
676 User *LoadUser = *I->user_begin();
677 if (isa<InsertElementInst>(LoadUser))
678 IsVectorAccess = true;
679 }
680
681 if (IsFPAccess || IsVectorAccess)
682 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
683 }
684
685 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
686}
687
688bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
689 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
690 // Punt on globals for now, although they can be used in limited
691 // RELATIVE LONG cases.
692 if (AM.BaseGV)
693 return false;
694
695 // Require a 20-bit signed offset.
696 if (!isInt<20>(AM.BaseOffs))
697 return false;
698
699 AddressingMode SupportedAM(true, true);
700 if (I != nullptr)
701 SupportedAM = supportedAddressingMode(I, Subtarget.hasVector());
702
703 if (!SupportedAM.LongDisplacement && !isUInt<12>(AM.BaseOffs))
704 return false;
705
706 if (!SupportedAM.IndexReg)
707 // No indexing allowed.
708 return AM.Scale == 0;
709 else
710 // Indexing is OK but no scale factor can be applied.
711 return AM.Scale == 0 || AM.Scale == 1;
712}
713
714bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
715 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
716 return false;
717 unsigned FromBits = FromType->getPrimitiveSizeInBits();
718 unsigned ToBits = ToType->getPrimitiveSizeInBits();
719 return FromBits > ToBits;
720}
721
722bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
723 if (!FromVT.isInteger() || !ToVT.isInteger())
724 return false;
725 unsigned FromBits = FromVT.getSizeInBits();
726 unsigned ToBits = ToVT.getSizeInBits();
727 return FromBits > ToBits;
728}
729
730//===----------------------------------------------------------------------===//
731// Inline asm support
732//===----------------------------------------------------------------------===//
733
734TargetLowering::ConstraintType
735SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
736 if (Constraint.size() == 1) {
737 switch (Constraint[0]) {
738 case 'a': // Address register
739 case 'd': // Data register (equivalent to 'r')
740 case 'f': // Floating-point register
741 case 'h': // High-part register
742 case 'r': // General-purpose register
743 return C_RegisterClass;
744
745 case 'Q': // Memory with base and unsigned 12-bit displacement
746 case 'R': // Likewise, plus an index
747 case 'S': // Memory with base and signed 20-bit displacement
748 case 'T': // Likewise, plus an index
749 case 'm': // Equivalent to 'T'.
750 return C_Memory;
751
752 case 'I': // Unsigned 8-bit constant
753 case 'J': // Unsigned 12-bit constant
754 case 'K': // Signed 16-bit constant
755 case 'L': // Signed 20-bit displacement (on all targets we support)
756 case 'M': // 0x7fffffff
757 return C_Other;
758
759 default:
760 break;
761 }
762 }
763 return TargetLowering::getConstraintType(Constraint);
764}
765
766TargetLowering::ConstraintWeight SystemZTargetLowering::
767getSingleConstraintMatchWeight(AsmOperandInfo &info,
768 const char *constraint) const {
769 ConstraintWeight weight = CW_Invalid;
770 Value *CallOperandVal = info.CallOperandVal;
771 // If we don't have a value, we can't do a match,
772 // but allow it at the lowest weight.
773 if (!CallOperandVal)
774 return CW_Default;
775 Type *type = CallOperandVal->getType();
776 // Look at the constraint type.
777 switch (*constraint) {
778 default:
779 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
780 break;
781
782 case 'a': // Address register
783 case 'd': // Data register (equivalent to 'r')
784 case 'h': // High-part register
785 case 'r': // General-purpose register
786 if (CallOperandVal->getType()->isIntegerTy())
787 weight = CW_Register;
788 break;
789
790 case 'f': // Floating-point register
791 if (type->isFloatingPointTy())
792 weight = CW_Register;
793 break;
794
795 case 'I': // Unsigned 8-bit constant
796 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
797 if (isUInt<8>(C->getZExtValue()))
798 weight = CW_Constant;
799 break;
800
801 case 'J': // Unsigned 12-bit constant
802 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
803 if (isUInt<12>(C->getZExtValue()))
804 weight = CW_Constant;
805 break;
806
807 case 'K': // Signed 16-bit constant
808 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
809 if (isInt<16>(C->getSExtValue()))
810 weight = CW_Constant;
811 break;
812
813 case 'L': // Signed 20-bit displacement (on all targets we support)
814 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
815 if (isInt<20>(C->getSExtValue()))
816 weight = CW_Constant;
817 break;
818
819 case 'M': // 0x7fffffff
820 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
821 if (C->getZExtValue() == 0x7fffffff)
822 weight = CW_Constant;
823 break;
824 }
825 return weight;
826}
827
828// Parse a "{tNNN}" register constraint for which the register type "t"
829// has already been verified. MC is the class associated with "t" and
830// Map maps 0-based register numbers to LLVM register numbers.
831static std::pair<unsigned, const TargetRegisterClass *>
832parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
833 const unsigned *Map) {
834 assert(*(Constraint.end()-1) == '}' && "Missing '}'")(static_cast <bool> (*(Constraint.end()-1) == '}' &&
"Missing '}'") ? void (0) : __assert_fail ("*(Constraint.end()-1) == '}' && \"Missing '}'\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 834, __extension__ __PRETTY_FUNCTION__))
;
835 if (isdigit(Constraint[2])) {
836 unsigned Index;
837 bool Failed =
838 Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index);
839 if (!Failed && Index < 16 && Map[Index])
840 return std::make_pair(Map[Index], RC);
841 }
842 return std::make_pair(0U, nullptr);
843}
844
845std::pair<unsigned, const TargetRegisterClass *>
846SystemZTargetLowering::getRegForInlineAsmConstraint(
847 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
848 if (Constraint.size() == 1) {
849 // GCC Constraint Letters
850 switch (Constraint[0]) {
851 default: break;
852 case 'd': // Data register (equivalent to 'r')
853 case 'r': // General-purpose register
854 if (VT == MVT::i64)
855 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
856 else if (VT == MVT::i128)
857 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
858 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
859
860 case 'a': // Address register
861 if (VT == MVT::i64)
862 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
863 else if (VT == MVT::i128)
864 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
865 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
866
867 case 'h': // High-part register (an LLVM extension)
868 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
869
870 case 'f': // Floating-point register
871 if (VT == MVT::f64)
872 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
873 else if (VT == MVT::f128)
874 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
875 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
876 }
877 }
878 if (Constraint.size() > 0 && Constraint[0] == '{') {
879 // We need to override the default register parsing for GPRs and FPRs
880 // because the interpretation depends on VT. The internal names of
881 // the registers are also different from the external names
882 // (F0D and F0S instead of F0, etc.).
883 if (Constraint[1] == 'r') {
884 if (VT == MVT::i32)
885 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass,
886 SystemZMC::GR32Regs);
887 if (VT == MVT::i128)
888 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass,
889 SystemZMC::GR128Regs);
890 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass,
891 SystemZMC::GR64Regs);
892 }
893 if (Constraint[1] == 'f') {
894 if (VT == MVT::f32)
895 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass,
896 SystemZMC::FP32Regs);
897 if (VT == MVT::f128)
898 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass,
899 SystemZMC::FP128Regs);
900 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass,
901 SystemZMC::FP64Regs);
902 }
903 }
904 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
905}
906
907void SystemZTargetLowering::
908LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
909 std::vector<SDValue> &Ops,
910 SelectionDAG &DAG) const {
911 // Only support length 1 constraints for now.
912 if (Constraint.length() == 1) {
913 switch (Constraint[0]) {
914 case 'I': // Unsigned 8-bit constant
915 if (auto *C = dyn_cast<ConstantSDNode>(Op))
916 if (isUInt<8>(C->getZExtValue()))
917 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
918 Op.getValueType()));
919 return;
920
921 case 'J': // Unsigned 12-bit constant
922 if (auto *C = dyn_cast<ConstantSDNode>(Op))
923 if (isUInt<12>(C->getZExtValue()))
924 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
925 Op.getValueType()));
926 return;
927
928 case 'K': // Signed 16-bit constant
929 if (auto *C = dyn_cast<ConstantSDNode>(Op))
930 if (isInt<16>(C->getSExtValue()))
931 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
932 Op.getValueType()));
933 return;
934
935 case 'L': // Signed 20-bit displacement (on all targets we support)
936 if (auto *C = dyn_cast<ConstantSDNode>(Op))
937 if (isInt<20>(C->getSExtValue()))
938 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
939 Op.getValueType()));
940 return;
941
942 case 'M': // 0x7fffffff
943 if (auto *C = dyn_cast<ConstantSDNode>(Op))
944 if (C->getZExtValue() == 0x7fffffff)
945 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
946 Op.getValueType()));
947 return;
948 }
949 }
950 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
951}
952
953//===----------------------------------------------------------------------===//
954// Calling conventions
955//===----------------------------------------------------------------------===//
956
957#include "SystemZGenCallingConv.inc"
958
959bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
960 Type *ToType) const {
961 return isTruncateFree(FromType, ToType);
962}
963
964bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
965 return CI->isTailCall();
966}
967
968// We do not yet support 128-bit single-element vector types. If the user
969// attempts to use such types as function argument or return type, prefer
970// to error out instead of emitting code violating the ABI.
971static void VerifyVectorType(MVT VT, EVT ArgVT) {
972 if (ArgVT.isVector() && !VT.isVector())
973 report_fatal_error("Unsupported vector argument or return type");
974}
975
976static void VerifyVectorTypes(const SmallVectorImpl<ISD::InputArg> &Ins) {
977 for (unsigned i = 0; i < Ins.size(); ++i)
978 VerifyVectorType(Ins[i].VT, Ins[i].ArgVT);
979}
980
981static void VerifyVectorTypes(const SmallVectorImpl<ISD::OutputArg> &Outs) {
982 for (unsigned i = 0; i < Outs.size(); ++i)
983 VerifyVectorType(Outs[i].VT, Outs[i].ArgVT);
984}
985
986// Value is a value that has been passed to us in the location described by VA
987// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
988// any loads onto Chain.
989static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
990 CCValAssign &VA, SDValue Chain,
991 SDValue Value) {
992 // If the argument has been promoted from a smaller type, insert an
993 // assertion to capture this.
994 if (VA.getLocInfo() == CCValAssign::SExt)
995 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value,
996 DAG.getValueType(VA.getValVT()));
997 else if (VA.getLocInfo() == CCValAssign::ZExt)
998 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value,
999 DAG.getValueType(VA.getValVT()));
1000
1001 if (VA.isExtInLoc())
1002 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value);
1003 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1004 // If this is a short vector argument loaded from the stack,
1005 // extend from i64 to full vector size and then bitcast.
1006 assert(VA.getLocVT() == MVT::i64)(static_cast <bool> (VA.getLocVT() == MVT::i64) ? void (
0) : __assert_fail ("VA.getLocVT() == MVT::i64", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1006, __extension__ __PRETTY_FUNCTION__))
;
1007 assert(VA.getValVT().isVector())(static_cast <bool> (VA.getValVT().isVector()) ? void (
0) : __assert_fail ("VA.getValVT().isVector()", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1007, __extension__ __PRETTY_FUNCTION__))
;
1008 Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)});
1009 Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
1010 } else
1011 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo")(static_cast <bool> (VA.getLocInfo() == CCValAssign::Full
&& "Unsupported getLocInfo") ? void (0) : __assert_fail
("VA.getLocInfo() == CCValAssign::Full && \"Unsupported getLocInfo\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1011, __extension__ __PRETTY_FUNCTION__))
;
1012 return Value;
1013}
1014
1015// Value is a value of type VA.getValVT() that we need to copy into
1016// the location described by VA. Return a copy of Value converted to
1017// VA.getValVT(). The caller is responsible for handling indirect values.
1018static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1019 CCValAssign &VA, SDValue Value) {
1020 switch (VA.getLocInfo()) {
1021 case CCValAssign::SExt:
1022 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value);
1023 case CCValAssign::ZExt:
1024 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
1025 case CCValAssign::AExt:
1026 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
1027 case CCValAssign::BCvt:
1028 // If this is a short vector argument to be stored to the stack,
1029 // bitcast to v2i64 and then extract first element.
1030 assert(VA.getLocVT() == MVT::i64)(static_cast <bool> (VA.getLocVT() == MVT::i64) ? void (
0) : __assert_fail ("VA.getLocVT() == MVT::i64", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1030, __extension__ __PRETTY_FUNCTION__))
;
1031 assert(VA.getValVT().isVector())(static_cast <bool> (VA.getValVT().isVector()) ? void (
0) : __assert_fail ("VA.getValVT().isVector()", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1031, __extension__ __PRETTY_FUNCTION__))
;
1032 Value = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Value);
1033 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
1034 DAG.getConstant(0, DL, MVT::i32));
1035 case CCValAssign::Full:
1036 return Value;
1037 default:
1038 llvm_unreachable("Unhandled getLocInfo()")::llvm::llvm_unreachable_internal("Unhandled getLocInfo()", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1038)
;
1039 }
1040}
1041
1042SDValue SystemZTargetLowering::LowerFormalArguments(
1043 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1044 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1045 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1046 MachineFunction &MF = DAG.getMachineFunction();
1047 MachineFrameInfo &MFI = MF.getFrameInfo();
1048 MachineRegisterInfo &MRI = MF.getRegInfo();
1049 SystemZMachineFunctionInfo *FuncInfo =
1050 MF.getInfo<SystemZMachineFunctionInfo>();
1051 auto *TFL =
1052 static_cast<const SystemZFrameLowering *>(Subtarget.getFrameLowering());
1053 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1054
1055 // Detect unsupported vector argument types.
1056 if (Subtarget.hasVector())
1057 VerifyVectorTypes(Ins);
1058
1059 // Assign locations to all of the incoming arguments.
1060 SmallVector<CCValAssign, 16> ArgLocs;
1061 SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1062 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ);
1063
1064 unsigned NumFixedGPRs = 0;
1065 unsigned NumFixedFPRs = 0;
1066 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1067 SDValue ArgValue;
1068 CCValAssign &VA = ArgLocs[I];
1069 EVT LocVT = VA.getLocVT();
1070 if (VA.isRegLoc()) {
1071 // Arguments passed in registers
1072 const TargetRegisterClass *RC;
1073 switch (LocVT.getSimpleVT().SimpleTy) {
1074 default:
1075 // Integers smaller than i64 should be promoted to i64.
1076 llvm_unreachable("Unexpected argument type")::llvm::llvm_unreachable_internal("Unexpected argument type",
"/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1076)
;
1077 case MVT::i32:
1078 NumFixedGPRs += 1;
1079 RC = &SystemZ::GR32BitRegClass;
1080 break;
1081 case MVT::i64:
1082 NumFixedGPRs += 1;
1083 RC = &SystemZ::GR64BitRegClass;
1084 break;
1085 case MVT::f32:
1086 NumFixedFPRs += 1;
1087 RC = &SystemZ::FP32BitRegClass;
1088 break;
1089 case MVT::f64:
1090 NumFixedFPRs += 1;
1091 RC = &SystemZ::FP64BitRegClass;
1092 break;
1093 case MVT::v16i8:
1094 case MVT::v8i16:
1095 case MVT::v4i32:
1096 case MVT::v2i64:
1097 case MVT::v4f32:
1098 case MVT::v2f64:
1099 RC = &SystemZ::VR128BitRegClass;
1100 break;
1101 }
1102
1103 unsigned VReg = MRI.createVirtualRegister(RC);
1104 MRI.addLiveIn(VA.getLocReg(), VReg);
1105 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1106 } else {
1107 assert(VA.isMemLoc() && "Argument not register or memory")(static_cast <bool> (VA.isMemLoc() && "Argument not register or memory"
) ? void (0) : __assert_fail ("VA.isMemLoc() && \"Argument not register or memory\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1107, __extension__ __PRETTY_FUNCTION__))
;
1108
1109 // Create the frame index object for this incoming parameter.
1110 int FI = MFI.CreateFixedObject(LocVT.getSizeInBits() / 8,
1111 VA.getLocMemOffset(), true);
1112
1113 // Create the SelectionDAG nodes corresponding to a load
1114 // from this parameter. Unpromoted ints and floats are
1115 // passed as right-justified 8-byte values.
1116 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1117 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
1118 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
1119 DAG.getIntPtrConstant(4, DL));
1120 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
1121 MachinePointerInfo::getFixedStack(MF, FI));
1122 }
1123
1124 // Convert the value of the argument register into the value that's
1125 // being passed.
1126 if (VA.getLocInfo() == CCValAssign::Indirect) {
1127 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1128 MachinePointerInfo()));
1129 // If the original argument was split (e.g. i128), we need
1130 // to load all parts of it here (using the same address).
1131 unsigned ArgIndex = Ins[I].OrigArgIndex;
1132 assert (Ins[I].PartOffset == 0)(static_cast <bool> (Ins[I].PartOffset == 0) ? void (0)
: __assert_fail ("Ins[I].PartOffset == 0", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1132, __extension__ __PRETTY_FUNCTION__))
;
1133 while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) {
1134 CCValAssign &PartVA = ArgLocs[I + 1];
1135 unsigned PartOffset = Ins[I + 1].PartOffset;
1136 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1137 DAG.getIntPtrConstant(PartOffset, DL));
1138 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1139 MachinePointerInfo()));
1140 ++I;
1141 }
1142 } else
1143 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
1144 }
1145
1146 if (IsVarArg) {
1147 // Save the number of non-varargs registers for later use by va_start, etc.
1148 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
1149 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
1150
1151 // Likewise the address (in the form of a frame index) of where the
1152 // first stack vararg would be. The 1-byte size here is arbitrary.
1153 int64_t StackSize = CCInfo.getNextStackOffset();
1154 FuncInfo->setVarArgsFrameIndex(MFI.CreateFixedObject(1, StackSize, true));
1155
1156 // ...and a similar frame index for the caller-allocated save area
1157 // that will be used to store the incoming registers.
1158 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea();
1159 unsigned RegSaveIndex = MFI.CreateFixedObject(1, RegSaveOffset, true);
1160 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
1161
1162 // Store the FPR varargs in the reserved frame slots. (We store the
1163 // GPRs as part of the prologue.)
1164 if (NumFixedFPRs < SystemZ::NumArgFPRs) {
1165 SDValue MemOps[SystemZ::NumArgFPRs];
1166 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) {
1167 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]);
1168 int FI = MFI.CreateFixedObject(8, RegSaveOffset + Offset, true);
1169 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1170 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I],
1171 &SystemZ::FP64BitRegClass);
1172 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
1173 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
1174 MachinePointerInfo::getFixedStack(MF, FI));
1175 }
1176 // Join the stores, which are independent of one another.
1177 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
1178 makeArrayRef(&MemOps[NumFixedFPRs],
1179 SystemZ::NumArgFPRs-NumFixedFPRs));
1180 }
1181 }
1182
1183 return Chain;
1184}
1185
1186static bool canUseSiblingCall(const CCState &ArgCCInfo,
1187 SmallVectorImpl<CCValAssign> &ArgLocs,
1188 SmallVectorImpl<ISD::OutputArg> &Outs) {
1189 // Punt if there are any indirect or stack arguments, or if the call
1190 // needs the callee-saved argument register R6, or if the call uses
1191 // the callee-saved register arguments SwiftSelf and SwiftError.
1192 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1193 CCValAssign &VA = ArgLocs[I];
1194 if (VA.getLocInfo() == CCValAssign::Indirect)
1195 return false;
1196 if (!VA.isRegLoc())
1197 return false;
1198 unsigned Reg = VA.getLocReg();
1199 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
1200 return false;
1201 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
1202 return false;
1203 }
1204 return true;
1205}
1206
1207SDValue
1208SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
1209 SmallVectorImpl<SDValue> &InVals) const {
1210 SelectionDAG &DAG = CLI.DAG;
1211 SDLoc &DL = CLI.DL;
1212 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1213 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1214 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1215 SDValue Chain = CLI.Chain;
1216 SDValue Callee = CLI.Callee;
1217 bool &IsTailCall = CLI.IsTailCall;
1218 CallingConv::ID CallConv = CLI.CallConv;
1219 bool IsVarArg = CLI.IsVarArg;
1220 MachineFunction &MF = DAG.getMachineFunction();
1221 EVT PtrVT = getPointerTy(MF.getDataLayout());
1222
1223 // Detect unsupported vector argument and return types.
1224 if (Subtarget.hasVector()) {
1225 VerifyVectorTypes(Outs);
1226 VerifyVectorTypes(Ins);
1227 }
1228
1229 // Analyze the operands of the call, assigning locations to each operand.
1230 SmallVector<CCValAssign, 16> ArgLocs;
1231 SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1232 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
1233
1234 // We don't support GuaranteedTailCallOpt, only automatically-detected
1235 // sibling calls.
1236 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
1237 IsTailCall = false;
1238
1239 // Get a count of how many bytes are to be pushed on the stack.
1240 unsigned NumBytes = ArgCCInfo.getNextStackOffset();
1241
1242 // Mark the start of the call.
1243 if (!IsTailCall)
1244 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
1245
1246 // Copy argument values to their designated locations.
1247 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
1248 SmallVector<SDValue, 8> MemOpChains;
1249 SDValue StackPtr;
1250 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1251 CCValAssign &VA = ArgLocs[I];
1252 SDValue ArgValue = OutVals[I];
1253
1254 if (VA.getLocInfo() == CCValAssign::Indirect) {
1255 // Store the argument in a stack slot and pass its address.
1256 SDValue SpillSlot = DAG.CreateStackTemporary(Outs[I].ArgVT);
1257 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1258 MemOpChains.push_back(
1259 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
1260 MachinePointerInfo::getFixedStack(MF, FI)));
1261 // If the original argument was split (e.g. i128), we need
1262 // to store all parts of it here (and pass just one address).
1263 unsigned ArgIndex = Outs[I].OrigArgIndex;
1264 assert (Outs[I].PartOffset == 0)(static_cast <bool> (Outs[I].PartOffset == 0) ? void (0
) : __assert_fail ("Outs[I].PartOffset == 0", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1264, __extension__ __PRETTY_FUNCTION__))
;
1265 while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
1266 SDValue PartValue = OutVals[I + 1];
1267 unsigned PartOffset = Outs[I + 1].PartOffset;
1268 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
1269 DAG.getIntPtrConstant(PartOffset, DL));
1270 MemOpChains.push_back(
1271 DAG.getStore(Chain, DL, PartValue, Address,
1272 MachinePointerInfo::getFixedStack(MF, FI)));
1273 ++I;
1274 }
1275 ArgValue = SpillSlot;
1276 } else
1277 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue);
1278
1279 if (VA.isRegLoc())
1280 // Queue up the argument copies and emit them at the end.
1281 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
1282 else {
1283 assert(VA.isMemLoc() && "Argument not register or memory")(static_cast <bool> (VA.isMemLoc() && "Argument not register or memory"
) ? void (0) : __assert_fail ("VA.isMemLoc() && \"Argument not register or memory\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1283, __extension__ __PRETTY_FUNCTION__))
;
1284
1285 // Work out the address of the stack slot. Unpromoted ints and
1286 // floats are passed as right-justified 8-byte values.
1287 if (!StackPtr.getNode())
1288 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT);
1289 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset();
1290 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
1291 Offset += 4;
1292 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
1293 DAG.getIntPtrConstant(Offset, DL));
1294
1295 // Emit the store.
1296 MemOpChains.push_back(
1297 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
1298 }
1299 }
1300
1301 // Join the stores, which are independent of one another.
1302 if (!MemOpChains.empty())
1303 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
1304
1305 // Accept direct calls by converting symbolic call addresses to the
1306 // associated Target* opcodes. Force %r1 to be used for indirect
1307 // tail calls.
1308 SDValue Glue;
1309 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1310 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
1311 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
1312 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
1313 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT);
1314 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
1315 } else if (IsTailCall) {
1316 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
1317 Glue = Chain.getValue(1);
1318 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType());
1319 }
1320
1321 // Build a sequence of copy-to-reg nodes, chained and glued together.
1322 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
1323 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
1324 RegsToPass[I].second, Glue);
1325 Glue = Chain.getValue(1);
1326 }
1327
1328 // The first call operand is the chain and the second is the target address.
1329 SmallVector<SDValue, 8> Ops;
1330 Ops.push_back(Chain);
1331 Ops.push_back(Callee);
1332
1333 // Add argument registers to the end of the list so that they are
1334 // known live into the call.
1335 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
1336 Ops.push_back(DAG.getRegister(RegsToPass[I].first,
1337 RegsToPass[I].second.getValueType()));
1338
1339 // Add a register mask operand representing the call-preserved registers.
1340 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
1341 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
1342 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1342, __extension__ __PRETTY_FUNCTION__))
;
1343 Ops.push_back(DAG.getRegisterMask(Mask));
1344
1345 // Glue the call to the argument copies, if any.
1346 if (Glue.getNode())
1347 Ops.push_back(Glue);
1348
1349 // Emit the call.
1350 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1351 if (IsTailCall)
1352 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
1353 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
1354 Glue = Chain.getValue(1);
1355
1356 // Mark the end of the call, which is glued to the call itself.
1357 Chain = DAG.getCALLSEQ_END(Chain,
1358 DAG.getConstant(NumBytes, DL, PtrVT, true),
1359 DAG.getConstant(0, DL, PtrVT, true),
1360 Glue, DL);
1361 Glue = Chain.getValue(1);
1362
1363 // Assign locations to each value returned by this call.
1364 SmallVector<CCValAssign, 16> RetLocs;
1365 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
1366 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ);
1367
1368 // Copy all of the result registers out of their specified physreg.
1369 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
1370 CCValAssign &VA = RetLocs[I];
1371
1372 // Copy the value out, gluing the copy to the end of the call sequence.
1373 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
1374 VA.getLocVT(), Glue);
1375 Chain = RetValue.getValue(1);
1376 Glue = RetValue.getValue(2);
1377
1378 // Convert the value of the return register into the value that's
1379 // being returned.
1380 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue));
1381 }
1382
1383 return Chain;
1384}
1385
1386bool SystemZTargetLowering::
1387CanLowerReturn(CallingConv::ID CallConv,
1388 MachineFunction &MF, bool isVarArg,
1389 const SmallVectorImpl<ISD::OutputArg> &Outs,
1390 LLVMContext &Context) const {
1391 // Detect unsupported vector return types.
1392 if (Subtarget.hasVector())
1393 VerifyVectorTypes(Outs);
1394
1395 // Special case that we cannot easily detect in RetCC_SystemZ since
1396 // i128 is not a legal type.
1397 for (auto &Out : Outs)
1398 if (Out.ArgVT == MVT::i128)
1399 return false;
1400
1401 SmallVector<CCValAssign, 16> RetLocs;
1402 CCState RetCCInfo(CallConv, isVarArg, MF, RetLocs, Context);
1403 return RetCCInfo.CheckReturn(Outs, RetCC_SystemZ);
1404}
1405
1406SDValue
1407SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
1408 bool IsVarArg,
1409 const SmallVectorImpl<ISD::OutputArg> &Outs,
1410 const SmallVectorImpl<SDValue> &OutVals,
1411 const SDLoc &DL, SelectionDAG &DAG) const {
1412 MachineFunction &MF = DAG.getMachineFunction();
1413
1414 // Detect unsupported vector return types.
1415 if (Subtarget.hasVector())
1416 VerifyVectorTypes(Outs);
1417
1418 // Assign locations to each returned value.
1419 SmallVector<CCValAssign, 16> RetLocs;
1420 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
1421 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ);
1422
1423 // Quick exit for void returns
1424 if (RetLocs.empty())
1425 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain);
1426
1427 // Copy the result values into the output registers.
1428 SDValue Glue;
1429 SmallVector<SDValue, 4> RetOps;
1430 RetOps.push_back(Chain);
1431 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
1432 CCValAssign &VA = RetLocs[I];
1433 SDValue RetValue = OutVals[I];
1434
1435 // Make the return register live on exit.
1436 assert(VA.isRegLoc() && "Can only return in registers!")(static_cast <bool> (VA.isRegLoc() && "Can only return in registers!"
) ? void (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1436, __extension__ __PRETTY_FUNCTION__))
;
1437
1438 // Promote the value as required.
1439 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue);
1440
1441 // Chain and glue the copies together.
1442 unsigned Reg = VA.getLocReg();
1443 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue);
1444 Glue = Chain.getValue(1);
1445 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT()));
1446 }
1447
1448 // Update chain and glue.
1449 RetOps[0] = Chain;
1450 if (Glue.getNode())
1451 RetOps.push_back(Glue);
1452
1453 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps);
1454}
1455
1456// Return true if Op is an intrinsic node with chain that returns the CC value
1457// as its only (other) argument. Provide the associated SystemZISD opcode and
1458// the mask of valid CC values if so.
1459static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
1460 unsigned &CCValid) {
1461 unsigned Id = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
1462 switch (Id) {
1463 case Intrinsic::s390_tbegin:
1464 Opcode = SystemZISD::TBEGIN;
1465 CCValid = SystemZ::CCMASK_TBEGIN;
1466 return true;
1467
1468 case Intrinsic::s390_tbegin_nofloat:
1469 Opcode = SystemZISD::TBEGIN_NOFLOAT;
1470 CCValid = SystemZ::CCMASK_TBEGIN;
1471 return true;
1472
1473 case Intrinsic::s390_tend:
1474 Opcode = SystemZISD::TEND;
1475 CCValid = SystemZ::CCMASK_TEND;
1476 return true;
1477
1478 default:
1479 return false;
1480 }
1481}
1482
1483// Return true if Op is an intrinsic node without chain that returns the
1484// CC value as its final argument. Provide the associated SystemZISD
1485// opcode and the mask of valid CC values if so.
1486static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
1487 unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1488 switch (Id) {
1489 case Intrinsic::s390_vpkshs:
1490 case Intrinsic::s390_vpksfs:
1491 case Intrinsic::s390_vpksgs:
1492 Opcode = SystemZISD::PACKS_CC;
1493 CCValid = SystemZ::CCMASK_VCMP;
1494 return true;
1495
1496 case Intrinsic::s390_vpklshs:
1497 case Intrinsic::s390_vpklsfs:
1498 case Intrinsic::s390_vpklsgs:
1499 Opcode = SystemZISD::PACKLS_CC;
1500 CCValid = SystemZ::CCMASK_VCMP;
1501 return true;
1502
1503 case Intrinsic::s390_vceqbs:
1504 case Intrinsic::s390_vceqhs:
1505 case Intrinsic::s390_vceqfs:
1506 case Intrinsic::s390_vceqgs:
1507 Opcode = SystemZISD::VICMPES;
1508 CCValid = SystemZ::CCMASK_VCMP;
1509 return true;
1510
1511 case Intrinsic::s390_vchbs:
1512 case Intrinsic::s390_vchhs:
1513 case Intrinsic::s390_vchfs:
1514 case Intrinsic::s390_vchgs:
1515 Opcode = SystemZISD::VICMPHS;
1516 CCValid = SystemZ::CCMASK_VCMP;
1517 return true;
1518
1519 case Intrinsic::s390_vchlbs:
1520 case Intrinsic::s390_vchlhs:
1521 case Intrinsic::s390_vchlfs:
1522 case Intrinsic::s390_vchlgs:
1523 Opcode = SystemZISD::VICMPHLS;
1524 CCValid = SystemZ::CCMASK_VCMP;
1525 return true;
1526
1527 case Intrinsic::s390_vtm:
1528 Opcode = SystemZISD::VTM;
1529 CCValid = SystemZ::CCMASK_VCMP;
1530 return true;
1531
1532 case Intrinsic::s390_vfaebs:
1533 case Intrinsic::s390_vfaehs:
1534 case Intrinsic::s390_vfaefs:
1535 Opcode = SystemZISD::VFAE_CC;
1536 CCValid = SystemZ::CCMASK_ANY;
1537 return true;
1538
1539 case Intrinsic::s390_vfaezbs:
1540 case Intrinsic::s390_vfaezhs:
1541 case Intrinsic::s390_vfaezfs:
1542 Opcode = SystemZISD::VFAEZ_CC;
1543 CCValid = SystemZ::CCMASK_ANY;
1544 return true;
1545
1546 case Intrinsic::s390_vfeebs:
1547 case Intrinsic::s390_vfeehs:
1548 case Intrinsic::s390_vfeefs:
1549 Opcode = SystemZISD::VFEE_CC;
1550 CCValid = SystemZ::CCMASK_ANY;
1551 return true;
1552
1553 case Intrinsic::s390_vfeezbs:
1554 case Intrinsic::s390_vfeezhs:
1555 case Intrinsic::s390_vfeezfs:
1556 Opcode = SystemZISD::VFEEZ_CC;
1557 CCValid = SystemZ::CCMASK_ANY;
1558 return true;
1559
1560 case Intrinsic::s390_vfenebs:
1561 case Intrinsic::s390_vfenehs:
1562 case Intrinsic::s390_vfenefs:
1563 Opcode = SystemZISD::VFENE_CC;
1564 CCValid = SystemZ::CCMASK_ANY;
1565 return true;
1566
1567 case Intrinsic::s390_vfenezbs:
1568 case Intrinsic::s390_vfenezhs:
1569 case Intrinsic::s390_vfenezfs:
1570 Opcode = SystemZISD::VFENEZ_CC;
1571 CCValid = SystemZ::CCMASK_ANY;
1572 return true;
1573
1574 case Intrinsic::s390_vistrbs:
1575 case Intrinsic::s390_vistrhs:
1576 case Intrinsic::s390_vistrfs:
1577 Opcode = SystemZISD::VISTR_CC;
1578 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
1579 return true;
1580
1581 case Intrinsic::s390_vstrcbs:
1582 case Intrinsic::s390_vstrchs:
1583 case Intrinsic::s390_vstrcfs:
1584 Opcode = SystemZISD::VSTRC_CC;
1585 CCValid = SystemZ::CCMASK_ANY;
1586 return true;
1587
1588 case Intrinsic::s390_vstrczbs:
1589 case Intrinsic::s390_vstrczhs:
1590 case Intrinsic::s390_vstrczfs:
1591 Opcode = SystemZISD::VSTRCZ_CC;
1592 CCValid = SystemZ::CCMASK_ANY;
1593 return true;
1594
1595 case Intrinsic::s390_vfcedbs:
1596 case Intrinsic::s390_vfcesbs:
1597 Opcode = SystemZISD::VFCMPES;
1598 CCValid = SystemZ::CCMASK_VCMP;
1599 return true;
1600
1601 case Intrinsic::s390_vfchdbs:
1602 case Intrinsic::s390_vfchsbs:
1603 Opcode = SystemZISD::VFCMPHS;
1604 CCValid = SystemZ::CCMASK_VCMP;
1605 return true;
1606
1607 case Intrinsic::s390_vfchedbs:
1608 case Intrinsic::s390_vfchesbs:
1609 Opcode = SystemZISD::VFCMPHES;
1610 CCValid = SystemZ::CCMASK_VCMP;
1611 return true;
1612
1613 case Intrinsic::s390_vftcidb:
1614 case Intrinsic::s390_vftcisb:
1615 Opcode = SystemZISD::VFTCI;
1616 CCValid = SystemZ::CCMASK_VCMP;
1617 return true;
1618
1619 case Intrinsic::s390_tdc:
1620 Opcode = SystemZISD::TDC;
1621 CCValid = SystemZ::CCMASK_TDC;
1622 return true;
1623
1624 default:
1625 return false;
1626 }
1627}
1628
1629// Emit an intrinsic with chain with a glued value instead of its CC result.
1630static SDValue emitIntrinsicWithChainAndGlue(SelectionDAG &DAG, SDValue Op,
1631 unsigned Opcode) {
1632 // Copy all operands except the intrinsic ID.
1633 unsigned NumOps = Op.getNumOperands();
1634 SmallVector<SDValue, 6> Ops;
1635 Ops.reserve(NumOps - 1);
1636 Ops.push_back(Op.getOperand(0));
1637 for (unsigned I = 2; I < NumOps; ++I)
1638 Ops.push_back(Op.getOperand(I));
1639
1640 assert(Op->getNumValues() == 2 && "Expected only CC result and chain")(static_cast <bool> (Op->getNumValues() == 2 &&
"Expected only CC result and chain") ? void (0) : __assert_fail
("Op->getNumValues() == 2 && \"Expected only CC result and chain\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__))
;
1641 SDVTList RawVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1642 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
1643 SDValue OldChain = SDValue(Op.getNode(), 1);
1644 SDValue NewChain = SDValue(Intr.getNode(), 0);
1645 DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain);
1646 return Intr;
1647}
1648
1649// Emit an intrinsic with a glued value instead of its CC result.
1650static SDValue emitIntrinsicWithGlue(SelectionDAG &DAG, SDValue Op,
1651 unsigned Opcode) {
1652 // Copy all operands except the intrinsic ID.
1653 unsigned NumOps = Op.getNumOperands();
1654 SmallVector<SDValue, 6> Ops;
1655 Ops.reserve(NumOps - 1);
1656 for (unsigned I = 1; I < NumOps; ++I)
1657 Ops.push_back(Op.getOperand(I));
1658
1659 if (Op->getNumValues() == 1)
1660 return DAG.getNode(Opcode, SDLoc(Op), MVT::Glue, Ops);
1661 assert(Op->getNumValues() == 2 && "Expected exactly one non-CC result")(static_cast <bool> (Op->getNumValues() == 2 &&
"Expected exactly one non-CC result") ? void (0) : __assert_fail
("Op->getNumValues() == 2 && \"Expected exactly one non-CC result\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1661, __extension__ __PRETTY_FUNCTION__))
;
1662 SDVTList RawVTs = DAG.getVTList(Op->getValueType(0), MVT::Glue);
1663 return DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
1664}
1665
1666// CC is a comparison that will be implemented using an integer or
1667// floating-point comparison. Return the condition code mask for
1668// a branch on true. In the integer case, CCMASK_CMP_UO is set for
1669// unsigned comparisons and clear for signed ones. In the floating-point
1670// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
1671static unsigned CCMaskForCondCode(ISD::CondCode CC) {
1672#define CONV(X) \
1673 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
1674 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
1675 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
1676
1677 switch (CC) {
1678 default:
1679 llvm_unreachable("Invalid integer condition!")::llvm::llvm_unreachable_internal("Invalid integer condition!"
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1679)
;
1680
1681 CONV(EQ);
1682 CONV(NE);
1683 CONV(GT);
1684 CONV(GE);
1685 CONV(LT);
1686 CONV(LE);
1687
1688 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
1689 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
1690 }
1691#undef CONV
1692}
1693
1694// If C can be converted to a comparison against zero, adjust the operands
1695// as necessary.
1696static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
1697 if (C.ICmpType == SystemZICMP::UnsignedOnly)
1698 return;
1699
1700 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode());
1701 if (!ConstOp1)
1702 return;
1703
1704 int64_t Value = ConstOp1->getSExtValue();
1705 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
1706 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
1707 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
1708 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
1709 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
1710 C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
1711 }
1712}
1713
1714// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
1715// adjust the operands as necessary.
1716static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
1717 Comparison &C) {
1718 // For us to make any changes, it must a comparison between a single-use
1719 // load and a constant.
1720 if (!C.Op0.hasOneUse() ||
1721 C.Op0.getOpcode() != ISD::LOAD ||
1722 C.Op1.getOpcode() != ISD::Constant)
1723 return;
1724
1725 // We must have an 8- or 16-bit load.
1726 auto *Load = cast<LoadSDNode>(C.Op0);
1727 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits();
1728 if (NumBits != 8 && NumBits != 16)
1729 return;
1730
1731 // The load must be an extending one and the constant must be within the
1732 // range of the unextended value.
1733 auto *ConstOp1 = cast<ConstantSDNode>(C.Op1);
1734 uint64_t Value = ConstOp1->getZExtValue();
1735 uint64_t Mask = (1 << NumBits) - 1;
1736 if (Load->getExtensionType() == ISD::SEXTLOAD) {
1737 // Make sure that ConstOp1 is in range of C.Op0.
1738 int64_t SignedValue = ConstOp1->getSExtValue();
1739 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
1740 return;
1741 if (C.ICmpType != SystemZICMP::SignedOnly) {
1742 // Unsigned comparison between two sign-extended values is equivalent
1743 // to unsigned comparison between two zero-extended values.
1744 Value &= Mask;
1745 } else if (NumBits == 8) {
1746 // Try to treat the comparison as unsigned, so that we can use CLI.
1747 // Adjust CCMask and Value as necessary.
1748 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
1749 // Test whether the high bit of the byte is set.
1750 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
1751 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
1752 // Test whether the high bit of the byte is clear.
1753 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
1754 else
1755 // No instruction exists for this combination.
1756 return;
1757 C.ICmpType = SystemZICMP::UnsignedOnly;
1758 }
1759 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
1760 if (Value > Mask)
1761 return;
1762 // If the constant is in range, we can use any comparison.
1763 C.ICmpType = SystemZICMP::Any;
1764 } else
1765 return;
1766
1767 // Make sure that the first operand is an i32 of the right extension type.
1768 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
1769 ISD::SEXTLOAD :
1770 ISD::ZEXTLOAD);
1771 if (C.Op0.getValueType() != MVT::i32 ||
1772 Load->getExtensionType() != ExtType) {
1773 C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(),
1774 Load->getBasePtr(), Load->getPointerInfo(),
1775 Load->getMemoryVT(), Load->getAlignment(),
1776 Load->getMemOperand()->getFlags());
1777 // Update the chain uses.
1778 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), C.Op0.getValue(1));
1779 }
1780
1781 // Make sure that the second operand is an i32 with the right value.
1782 if (C.Op1.getValueType() != MVT::i32 ||
1783 Value != ConstOp1->getZExtValue())
1784 C.Op1 = DAG.getConstant(Value, DL, MVT::i32);
1785}
1786
1787// Return true if Op is either an unextended load, or a load suitable
1788// for integer register-memory comparisons of type ICmpType.
1789static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
1790 auto *Load = dyn_cast<LoadSDNode>(Op.getNode());
1791 if (Load) {
1792 // There are no instructions to compare a register with a memory byte.
1793 if (Load->getMemoryVT() == MVT::i8)
1794 return false;
1795 // Otherwise decide on extension type.
1796 switch (Load->getExtensionType()) {
1797 case ISD::NON_EXTLOAD:
1798 return true;
1799 case ISD::SEXTLOAD:
1800 return ICmpType != SystemZICMP::UnsignedOnly;
1801 case ISD::ZEXTLOAD:
1802 return ICmpType != SystemZICMP::SignedOnly;
1803 default:
1804 break;
1805 }
1806 }
1807 return false;
1808}
1809
1810// Return true if it is better to swap the operands of C.
1811static bool shouldSwapCmpOperands(const Comparison &C) {
1812 // Leave f128 comparisons alone, since they have no memory forms.
1813 if (C.Op0.getValueType() == MVT::f128)
1814 return false;
1815
1816 // Always keep a floating-point constant second, since comparisons with
1817 // zero can use LOAD TEST and comparisons with other constants make a
1818 // natural memory operand.
1819 if (isa<ConstantFPSDNode>(C.Op1))
1820 return false;
1821
1822 // Never swap comparisons with zero since there are many ways to optimize
1823 // those later.
1824 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
1825 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
1826 return false;
1827
1828 // Also keep natural memory operands second if the loaded value is
1829 // only used here. Several comparisons have memory forms.
1830 if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse())
1831 return false;
1832
1833 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
1834 // In that case we generally prefer the memory to be second.
1835 if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) {
1836 // The only exceptions are when the second operand is a constant and
1837 // we can use things like CHHSI.
1838 if (!ConstOp1)
1839 return true;
1840 // The unsigned memory-immediate instructions can handle 16-bit
1841 // unsigned integers.
1842 if (C.ICmpType != SystemZICMP::SignedOnly &&
1843 isUInt<16>(ConstOp1->getZExtValue()))
1844 return false;
1845 // The signed memory-immediate instructions can handle 16-bit
1846 // signed integers.
1847 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
1848 isInt<16>(ConstOp1->getSExtValue()))
1849 return false;
1850 return true;
1851 }
1852
1853 // Try to promote the use of CGFR and CLGFR.
1854 unsigned Opcode0 = C.Op0.getOpcode();
1855 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
1856 return true;
1857 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
1858 return true;
1859 if (C.ICmpType != SystemZICMP::SignedOnly &&
1860 Opcode0 == ISD::AND &&
1861 C.Op0.getOperand(1).getOpcode() == ISD::Constant &&
1862 cast<ConstantSDNode>(C.Op0.getOperand(1))->getZExtValue() == 0xffffffff)
1863 return true;
1864
1865 return false;
1866}
1867
1868// Return a version of comparison CC mask CCMask in which the LT and GT
1869// actions are swapped.
1870static unsigned reverseCCMask(unsigned CCMask) {
1871 return ((CCMask & SystemZ::CCMASK_CMP_EQ) |
1872 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) |
1873 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) |
1874 (CCMask & SystemZ::CCMASK_CMP_UO));
1875}
1876
1877// Check whether C tests for equality between X and Y and whether X - Y
1878// or Y - X is also computed. In that case it's better to compare the
1879// result of the subtraction against zero.
1880static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
1881 Comparison &C) {
1882 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
1883 C.CCMask == SystemZ::CCMASK_CMP_NE) {
1884 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) {
1885 SDNode *N = *I;
1886 if (N->getOpcode() == ISD::SUB &&
1887 ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) ||
1888 (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {
1889 C.Op0 = SDValue(N, 0);
1890 C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
1891 return;
1892 }
1893 }
1894 }
1895}
1896
1897// Check whether C compares a floating-point value with zero and if that
1898// floating-point value is also negated. In this case we can use the
1899// negation to set CC, so avoiding separate LOAD AND TEST and
1900// LOAD (NEGATIVE/COMPLEMENT) instructions.
1901static void adjustForFNeg(Comparison &C) {
1902 auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1);
1903 if (C1 && C1->isZero()) {
1904 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) {
1905 SDNode *N = *I;
1906 if (N->getOpcode() == ISD::FNEG) {
1907 C.Op0 = SDValue(N, 0);
1908 C.CCMask = reverseCCMask(C.CCMask);
1909 return;
1910 }
1911 }
1912 }
1913}
1914
1915// Check whether C compares (shl X, 32) with 0 and whether X is
1916// also sign-extended. In that case it is better to test the result
1917// of the sign extension using LTGFR.
1918//
1919// This case is important because InstCombine transforms a comparison
1920// with (sext (trunc X)) into a comparison with (shl X, 32).
1921static void adjustForLTGFR(Comparison &C) {
1922 // Check for a comparison between (shl X, 32) and 0.
1923 if (C.Op0.getOpcode() == ISD::SHL &&
1924 C.Op0.getValueType() == MVT::i64 &&
1925 C.Op1.getOpcode() == ISD::Constant &&
1926 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
1927 auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
1928 if (C1 && C1->getZExtValue() == 32) {
1929 SDValue ShlOp0 = C.Op0.getOperand(0);
1930 // See whether X has any SIGN_EXTEND_INREG uses.
1931 for (auto I = ShlOp0->use_begin(), E = ShlOp0->use_end(); I != E; ++I) {
1932 SDNode *N = *I;
1933 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
1934 cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) {
1935 C.Op0 = SDValue(N, 0);
1936 return;
1937 }
1938 }
1939 }
1940 }
1941}
1942
1943// If C compares the truncation of an extending load, try to compare
1944// the untruncated value instead. This exposes more opportunities to
1945// reuse CC.
1946static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
1947 Comparison &C) {
1948 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
1949 C.Op0.getOperand(0).getOpcode() == ISD::LOAD &&
1950 C.Op1.getOpcode() == ISD::Constant &&
1951 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
1952 auto *L = cast<LoadSDNode>(C.Op0.getOperand(0));
1953 if (L->getMemoryVT().getStoreSizeInBits() <= C.Op0.getValueSizeInBits()) {
1954 unsigned Type = L->getExtensionType();
1955 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
1956 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
1957 C.Op0 = C.Op0.getOperand(0);
1958 C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
1959 }
1960 }
1961 }
1962}
1963
1964// Return true if shift operation N has an in-range constant shift value.
1965// Store it in ShiftVal if so.
1966static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
1967 auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1));
1968 if (!Shift)
1969 return false;
1970
1971 uint64_t Amount = Shift->getZExtValue();
1972 if (Amount >= N.getValueSizeInBits())
1973 return false;
1974
1975 ShiftVal = Amount;
1976 return true;
1977}
1978
1979// Check whether an AND with Mask is suitable for a TEST UNDER MASK
1980// instruction and whether the CC value is descriptive enough to handle
1981// a comparison of type Opcode between the AND result and CmpVal.
1982// CCMask says which comparison result is being tested and BitSize is
1983// the number of bits in the operands. If TEST UNDER MASK can be used,
1984// return the corresponding CC mask, otherwise return 0.
1985static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
1986 uint64_t Mask, uint64_t CmpVal,
1987 unsigned ICmpType) {
1988 assert(Mask != 0 && "ANDs with zero should have been removed by now")(static_cast <bool> (Mask != 0 && "ANDs with zero should have been removed by now"
) ? void (0) : __assert_fail ("Mask != 0 && \"ANDs with zero should have been removed by now\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 1988, __extension__ __PRETTY_FUNCTION__))
;
1989
1990 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
1991 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) &&
1992 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask))
1993 return 0;
1994
1995 // Work out the masks for the lowest and highest bits.
1996 unsigned HighShift = 63 - countLeadingZeros(Mask);
1997 uint64_t High = uint64_t(1) << HighShift;
1998 uint64_t Low = uint64_t(1) << countTrailingZeros(Mask);
1999
2000 // Signed ordered comparisons are effectively unsigned if the sign
2001 // bit is dropped.
2002 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
2003
2004 // Check for equality comparisons with 0, or the equivalent.
2005 if (CmpVal == 0) {
2006 if (CCMask == SystemZ::CCMASK_CMP_EQ)
2007 return SystemZ::CCMASK_TM_ALL_0;
2008 if (CCMask == SystemZ::CCMASK_CMP_NE)
2009 return SystemZ::CCMASK_TM_SOME_1;
2010 }
2011 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
2012 if (CCMask == SystemZ::CCMASK_CMP_LT)
2013 return SystemZ::CCMASK_TM_ALL_0;
2014 if (CCMask == SystemZ::CCMASK_CMP_GE)
2015 return SystemZ::CCMASK_TM_SOME_1;
2016 }
2017 if (EffectivelyUnsigned && CmpVal < Low) {
2018 if (CCMask == SystemZ::CCMASK_CMP_LE)
2019 return SystemZ::CCMASK_TM_ALL_0;
2020 if (CCMask == SystemZ::CCMASK_CMP_GT)
2021 return SystemZ::CCMASK_TM_SOME_1;
2022 }
2023
2024 // Check for equality comparisons with the mask, or the equivalent.
2025 if (CmpVal == Mask) {
2026 if (CCMask == SystemZ::CCMASK_CMP_EQ)
2027 return SystemZ::CCMASK_TM_ALL_1;
2028 if (CCMask == SystemZ::CCMASK_CMP_NE)
2029 return SystemZ::CCMASK_TM_SOME_0;
2030 }
2031 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
2032 if (CCMask == SystemZ::CCMASK_CMP_GT)
2033 return SystemZ::CCMASK_TM_ALL_1;
2034 if (CCMask == SystemZ::CCMASK_CMP_LE)
2035 return SystemZ::CCMASK_TM_SOME_0;
2036 }
2037 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
2038 if (CCMask == SystemZ::CCMASK_CMP_GE)
2039 return SystemZ::CCMASK_TM_ALL_1;
2040 if (CCMask == SystemZ::CCMASK_CMP_LT)
2041 return SystemZ::CCMASK_TM_SOME_0;
2042 }
2043
2044 // Check for ordered comparisons with the top bit.
2045 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
2046 if (CCMask == SystemZ::CCMASK_CMP_LE)
2047 return SystemZ::CCMASK_TM_MSB_0;
2048 if (CCMask == SystemZ::CCMASK_CMP_GT)
2049 return SystemZ::CCMASK_TM_MSB_1;
2050 }
2051 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
2052 if (CCMask == SystemZ::CCMASK_CMP_LT)
2053 return SystemZ::CCMASK_TM_MSB_0;
2054 if (CCMask == SystemZ::CCMASK_CMP_GE)
2055 return SystemZ::CCMASK_TM_MSB_1;
2056 }
2057
2058 // If there are just two bits, we can do equality checks for Low and High
2059 // as well.
2060 if (Mask == Low + High) {
2061 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
2062 return SystemZ::CCMASK_TM_MIXED_MSB_0;
2063 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
2064 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
2065 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
2066 return SystemZ::CCMASK_TM_MIXED_MSB_1;
2067 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
2068 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
2069 }
2070
2071 // Looks like we've exhausted our options.
2072 return 0;
2073}
2074
2075// See whether C can be implemented as a TEST UNDER MASK instruction.
2076// Update the arguments with the TM version if so.
2077static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
2078 Comparison &C) {
2079 // Check that we have a comparison with a constant.
2080 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
2081 if (!ConstOp1)
2082 return;
2083 uint64_t CmpVal = ConstOp1->getZExtValue();
2084
2085 // Check whether the nonconstant input is an AND with a constant mask.
2086 Comparison NewC(C);
2087 uint64_t MaskVal;
2088 ConstantSDNode *Mask = nullptr;
2089 if (C.Op0.getOpcode() == ISD::AND) {
2090 NewC.Op0 = C.Op0.getOperand(0);
2091 NewC.Op1 = C.Op0.getOperand(1);
2092 Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
2093 if (!Mask)
2094 return;
2095 MaskVal = Mask->getZExtValue();
2096 } else {
2097 // There is no instruction to compare with a 64-bit immediate
2098 // so use TMHH instead if possible. We need an unsigned ordered
2099 // comparison with an i64 immediate.
2100 if (NewC.Op0.getValueType() != MVT::i64 ||
2101 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
2102 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
2103 NewC.ICmpType == SystemZICMP::SignedOnly)
2104 return;
2105 // Convert LE and GT comparisons into LT and GE.
2106 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
2107 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
2108 if (CmpVal == uint64_t(-1))
2109 return;
2110 CmpVal += 1;
2111 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2112 }
2113 // If the low N bits of Op1 are zero than the low N bits of Op0 can
2114 // be masked off without changing the result.
2115 MaskVal = -(CmpVal & -CmpVal);
2116 NewC.ICmpType = SystemZICMP::UnsignedOnly;
2117 }
2118 if (!MaskVal)
2119 return;
2120
2121 // Check whether the combination of mask, comparison value and comparison
2122 // type are suitable.
2123 unsigned BitSize = NewC.Op0.getValueSizeInBits();
2124 unsigned NewCCMask, ShiftVal;
2125 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
2126 NewC.Op0.getOpcode() == ISD::SHL &&
2127 isSimpleShift(NewC.Op0, ShiftVal) &&
2128 (MaskVal >> ShiftVal != 0) &&
2129 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
2130 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
2131 MaskVal >> ShiftVal,
2132 CmpVal >> ShiftVal,
2133 SystemZICMP::Any))) {
2134 NewC.Op0 = NewC.Op0.getOperand(0);
2135 MaskVal >>= ShiftVal;
2136 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
2137 NewC.Op0.getOpcode() == ISD::SRL &&
2138 isSimpleShift(NewC.Op0, ShiftVal) &&
2139 (MaskVal << ShiftVal != 0) &&
2140 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
2141 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
2142 MaskVal << ShiftVal,
2143 CmpVal << ShiftVal,
2144 SystemZICMP::UnsignedOnly))) {
2145 NewC.Op0 = NewC.Op0.getOperand(0);
2146 MaskVal <<= ShiftVal;
2147 } else {
2148 NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal,
2149 NewC.ICmpType);
2150 if (!NewCCMask)
2151 return;
2152 }
2153
2154 // Go ahead and make the change.
2155 C.Opcode = SystemZISD::TM;
2156 C.Op0 = NewC.Op0;
2157 if (Mask && Mask->getZExtValue() == MaskVal)
2158 C.Op1 = SDValue(Mask, 0);
2159 else
2160 C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
2161 C.CCValid = SystemZ::CCMASK_TM;
2162 C.CCMask = NewCCMask;
2163}
2164
2165// See whether the comparison argument contains a redundant AND
2166// and remove it if so. This sometimes happens due to the generic
2167// BRCOND expansion.
2168static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
2169 Comparison &C) {
2170 if (C.Op0.getOpcode() != ISD::AND)
2171 return;
2172 auto *Mask = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
2173 if (!Mask)
2174 return;
2175 KnownBits Known;
2176 DAG.computeKnownBits(C.Op0.getOperand(0), Known);
2177 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
2178 return;
2179
2180 C.Op0 = C.Op0.getOperand(0);
2181}
2182
2183// Return a Comparison that tests the condition-code result of intrinsic
2184// node Call against constant integer CC using comparison code Cond.
2185// Opcode is the opcode of the SystemZISD operation for the intrinsic
2186// and CCValid is the set of possible condition-code results.
2187static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
2188 SDValue Call, unsigned CCValid, uint64_t CC,
2189 ISD::CondCode Cond) {
2190 Comparison C(Call, SDValue());
2191 C.Opcode = Opcode;
2192 C.CCValid = CCValid;
2193 if (Cond == ISD::SETEQ)
2194 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
2195 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
2196 else if (Cond == ISD::SETNE)
2197 // ...and the inverse of that.
2198 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
2199 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
2200 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
2201 // always true for CC>3.
2202 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
2203 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
2204 // ...and the inverse of that.
2205 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
2206 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
2207 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
2208 // always true for CC>3.
2209 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
2210 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
2211 // ...and the inverse of that.
2212 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
2213 else
2214 llvm_unreachable("Unexpected integer comparison type")::llvm::llvm_unreachable_internal("Unexpected integer comparison type"
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 2214)
;
2215 C.CCMask &= CCValid;
2216 return C;
2217}
2218
2219// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
2220static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
2221 ISD::CondCode Cond, const SDLoc &DL) {
2222 if (CmpOp1.getOpcode() == ISD::Constant) {
2223 uint64_t Constant = cast<ConstantSDNode>(CmpOp1)->getZExtValue();
2224 unsigned Opcode, CCValid;
2225 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
2226 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) &&
2227 isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid))
2228 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
2229 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
2230 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
2231 isIntrinsicWithCC(CmpOp0, Opcode, CCValid))
2232 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
2233 }
2234 Comparison C(CmpOp0, CmpOp1);
2235 C.CCMask = CCMaskForCondCode(Cond);
2236 if (C.Op0.getValueType().isFloatingPoint()) {
2237 C.CCValid = SystemZ::CCMASK_FCMP;
2238 C.Opcode = SystemZISD::FCMP;
2239 adjustForFNeg(C);
2240 } else {
2241 C.CCValid = SystemZ::CCMASK_ICMP;
2242 C.Opcode = SystemZISD::ICMP;
2243 // Choose the type of comparison. Equality and inequality tests can
2244 // use either signed or unsigned comparisons. The choice also doesn't
2245 // matter if both sign bits are known to be clear. In those cases we
2246 // want to give the main isel code the freedom to choose whichever
2247 // form fits best.
2248 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2249 C.CCMask == SystemZ::CCMASK_CMP_NE ||
2250 (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1)))
2251 C.ICmpType = SystemZICMP::Any;
2252 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
2253 C.ICmpType = SystemZICMP::UnsignedOnly;
2254 else
2255 C.ICmpType = SystemZICMP::SignedOnly;
2256 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
2257 adjustForRedundantAnd(DAG, DL, C);
2258 adjustZeroCmp(DAG, DL, C);
2259 adjustSubwordCmp(DAG, DL, C);
2260 adjustForSubtraction(DAG, DL, C);
2261 adjustForLTGFR(C);
2262 adjustICmpTruncate(DAG, DL, C);
2263 }
2264
2265 if (shouldSwapCmpOperands(C)) {
2266 std::swap(C.Op0, C.Op1);
2267 C.CCMask = reverseCCMask(C.CCMask);
2268 }
2269
2270 adjustForTestUnderMask(DAG, DL, C);
2271 return C;
2272}
2273
2274// Emit the comparison instruction described by C.
2275static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2276 if (!C.Op1.getNode()) {
2277 SDValue Op;
2278 switch (C.Op0.getOpcode()) {
2279 case ISD::INTRINSIC_W_CHAIN:
2280 Op = emitIntrinsicWithChainAndGlue(DAG, C.Op0, C.Opcode);
2281 break;
2282 case ISD::INTRINSIC_WO_CHAIN:
2283 Op = emitIntrinsicWithGlue(DAG, C.Op0, C.Opcode);
2284 break;
2285 default:
2286 llvm_unreachable("Invalid comparison operands")::llvm::llvm_unreachable_internal("Invalid comparison operands"
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 2286)
;
2287 }
2288 return SDValue(Op.getNode(), Op->getNumValues() - 1);
2289 }
2290 if (C.Opcode == SystemZISD::ICMP)
2291 return DAG.getNode(SystemZISD::ICMP, DL, MVT::Glue, C.Op0, C.Op1,
2292 DAG.getConstant(C.ICmpType, DL, MVT::i32));
2293 if (C.Opcode == SystemZISD::TM) {
2294 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
2295 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
2296 return DAG.getNode(SystemZISD::TM, DL, MVT::Glue, C.Op0, C.Op1,
2297 DAG.getConstant(RegisterOnly, DL, MVT::i32));
2298 }
2299 return DAG.getNode(C.Opcode, DL, MVT::Glue, C.Op0, C.Op1);
2300}
2301
2302// Implement a 32-bit *MUL_LOHI operation by extending both operands to
2303// 64 bits. Extend is the extension type to use. Store the high part
2304// in Hi and the low part in Lo.
2305static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
2306 SDValue Op0, SDValue Op1, SDValue &Hi,
2307 SDValue &Lo) {
2308 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
2309 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
2310 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
2311 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2312 DAG.getConstant(32, DL, MVT::i64));
2313 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
2314 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
2315}
2316
2317// Lower a binary operation that produces two VT results, one in each
2318// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
2319// and Opcode performs the GR128 operation. Store the even register result
2320// in Even and the odd register result in Odd.
2321static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
2322 unsigned Opcode, SDValue Op0, SDValue Op1,
2323 SDValue &Even, SDValue &Odd) {
2324 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1);
2325 bool Is32Bit = is32Bit(VT);
2326 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
2327 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
2328}
2329
2330// Return an i32 value that is 1 if the CC value produced by Glue is
2331// in the mask CCMask and 0 otherwise. CC is known to have a value
2332// in CCValid, so other values can be ignored.
2333static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue Glue,
2334 unsigned CCValid, unsigned CCMask) {
2335 SDValue Ops[] = { DAG.getConstant(1, DL, MVT::i32),
2336 DAG.getConstant(0, DL, MVT::i32),
2337 DAG.getConstant(CCValid, DL, MVT::i32),
2338 DAG.getConstant(CCMask, DL, MVT::i32), Glue };
2339 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
2340}
2341
2342// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
2343// be done directly. IsFP is true if CC is for a floating-point rather than
2344// integer comparison.
2345static unsigned getVectorComparison(ISD::CondCode CC, bool IsFP) {
2346 switch (CC) {
2347 case ISD::SETOEQ:
2348 case ISD::SETEQ:
2349 return IsFP ? SystemZISD::VFCMPE : SystemZISD::VICMPE;
2350
2351 case ISD::SETOGE:
2352 case ISD::SETGE:
2353 return IsFP ? SystemZISD::VFCMPHE : static_cast<SystemZISD::NodeType>(0);
2354
2355 case ISD::SETOGT:
2356 case ISD::SETGT:
2357 return IsFP ? SystemZISD::VFCMPH : SystemZISD::VICMPH;
2358
2359 case ISD::SETUGT:
2360 return IsFP ? static_cast<SystemZISD::NodeType>(0) : SystemZISD::VICMPHL;
2361
2362 default:
2363 return 0;
2364 }
2365}
2366
2367// Return the SystemZISD vector comparison operation for CC or its inverse,
2368// or 0 if neither can be done directly. Indicate in Invert whether the
2369// result is for the inverse of CC. IsFP is true if CC is for a
2370// floating-point rather than integer comparison.
2371static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, bool IsFP,
2372 bool &Invert) {
2373 if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
2374 Invert = false;
2375 return Opcode;
2376 }
2377
2378 CC = ISD::getSetCCInverse(CC, !IsFP);
2379 if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
2380 Invert = true;
2381 return Opcode;
2382 }
2383
2384 return 0;
2385}
2386
2387// Return a v2f64 that contains the extended form of elements Start and Start+1
2388// of v4f32 value Op.
2389static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
2390 SDValue Op) {
2391 int Mask[] = { Start, -1, Start + 1, -1 };
2392 Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
2393 return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
2394}
2395
2396// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
2397// producing a result of type VT.
2398SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
2399 const SDLoc &DL, EVT VT,
2400 SDValue CmpOp0,
2401 SDValue CmpOp1) const {
2402 // There is no hardware support for v4f32 (unless we have the vector
2403 // enhancements facility 1), so extend the vector into two v2f64s
2404 // and compare those.
2405 if (CmpOp0.getValueType() == MVT::v4f32 &&
2406 !Subtarget.hasVectorEnhancements1()) {
2407 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0);
2408 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0);
2409 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1);
2410 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1);
2411 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
2412 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
2413 return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
2414 }
2415 return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
2416}
2417
2418// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
2419// an integer mask of type VT.
2420SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
2421 const SDLoc &DL, EVT VT,
2422 ISD::CondCode CC,
2423 SDValue CmpOp0,
2424 SDValue CmpOp1) const {
2425 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
2426 bool Invert = false;
2427 SDValue Cmp;
2428 switch (CC) {
2429 // Handle tests for order using (or (ogt y x) (oge x y)).
2430 case ISD::SETUO:
2431 Invert = true;
2432 LLVM_FALLTHROUGH[[clang::fallthrough]];
2433 case ISD::SETO: {
2434 assert(IsFP && "Unexpected integer comparison")(static_cast <bool> (IsFP && "Unexpected integer comparison"
) ? void (0) : __assert_fail ("IsFP && \"Unexpected integer comparison\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 2434, __extension__ __PRETTY_FUNCTION__))
;
2435 SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
2436 SDValue GE = getVectorCmp(DAG, SystemZISD::VFCMPHE, DL, VT, CmpOp0, CmpOp1);
2437 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
2438 break;
2439 }
2440
2441 // Handle <> tests using (or (ogt y x) (ogt x y)).
2442 case ISD::SETUEQ:
2443 Invert = true;
2444 LLVM_FALLTHROUGH[[clang::fallthrough]];
2445 case ISD::SETONE: {
2446 assert(IsFP && "Unexpected integer comparison")(static_cast <bool> (IsFP && "Unexpected integer comparison"
) ? void (0) : __assert_fail ("IsFP && \"Unexpected integer comparison\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 2446, __extension__ __PRETTY_FUNCTION__))
;
2447 SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
2448 SDValue GT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp0, CmpOp1);
2449 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
2450 break;
2451 }
2452
2453 // Otherwise a single comparison is enough. It doesn't really
2454 // matter whether we try the inversion or the swap first, since
2455 // there are no cases where both work.
2456 default:
2457 if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
2458 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1);
2459 else {
2460 CC = ISD::getSetCCSwappedOperands(CC);
2461 if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
2462 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0);
2463 else
2464 llvm_unreachable("Unhandled comparison")::llvm::llvm_unreachable_internal("Unhandled comparison", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 2464)
;
2465 }
2466 break;
2467 }
2468 if (Invert) {
2469 SDValue Mask = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
2470 DAG.getConstant(65535, DL, MVT::i32));
2471 Mask = DAG.getNode(ISD::BITCAST, DL, VT, Mask);
2472 Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
2473 }
2474 return Cmp;
2475}
2476
2477SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
2478 SelectionDAG &DAG) const {
2479 SDValue CmpOp0 = Op.getOperand(0);
2480 SDValue CmpOp1 = Op.getOperand(1);
2481 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
2482 SDLoc DL(Op);
2483 EVT VT = Op.getValueType();
2484 if (VT.isVector())
2485 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
2486
2487 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2488 SDValue Glue = emitCmp(DAG, DL, C);
2489 return emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask);
2490}
2491
2492SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
2493 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
2494 SDValue CmpOp0 = Op.getOperand(2);
2495 SDValue CmpOp1 = Op.getOperand(3);
2496 SDValue Dest = Op.getOperand(4);
2497 SDLoc DL(Op);
2498
2499 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2500 SDValue Glue = emitCmp(DAG, DL, C);
2501 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(),
2502 Op.getOperand(0), DAG.getConstant(C.CCValid, DL, MVT::i32),
2503 DAG.getConstant(C.CCMask, DL, MVT::i32), Dest, Glue);
2504}
2505
2506// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
2507// allowing Pos and Neg to be wider than CmpOp.
2508static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
2509 return (Neg.getOpcode() == ISD::SUB &&
2510 Neg.getOperand(0).getOpcode() == ISD::Constant &&
2511 cast<ConstantSDNode>(Neg.getOperand(0))->getZExtValue() == 0 &&
2512 Neg.getOperand(1) == Pos &&
2513 (Pos == CmpOp ||
2514 (Pos.getOpcode() == ISD::SIGN_EXTEND &&
2515 Pos.getOperand(0) == CmpOp)));
2516}
2517
2518// Return the absolute or negative absolute of Op; IsNegative decides which.
2519static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
2520 bool IsNegative) {
2521 Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op);
2522 if (IsNegative)
2523 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
2524 DAG.getConstant(0, DL, Op.getValueType()), Op);
2525 return Op;
2526}
2527
2528SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
2529 SelectionDAG &DAG) const {
2530 SDValue CmpOp0 = Op.getOperand(0);
2531 SDValue CmpOp1 = Op.getOperand(1);
2532 SDValue TrueOp = Op.getOperand(2);
2533 SDValue FalseOp = Op.getOperand(3);
2534 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
2535 SDLoc DL(Op);
2536
2537 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2538
2539 // Check for absolute and negative-absolute selections, including those
2540 // where the comparison value is sign-extended (for LPGFR and LNGFR).
2541 // This check supplements the one in DAGCombiner.
2542 if (C.Opcode == SystemZISD::ICMP &&
2543 C.CCMask != SystemZ::CCMASK_CMP_EQ &&
2544 C.CCMask != SystemZ::CCMASK_CMP_NE &&
2545 C.Op1.getOpcode() == ISD::Constant &&
2546 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
2547 if (isAbsolute(C.Op0, TrueOp, FalseOp))
2548 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
2549 if (isAbsolute(C.Op0, FalseOp, TrueOp))
2550 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
2551 }
2552
2553 SDValue Glue = emitCmp(DAG, DL, C);
2554 SDValue Ops[] = {TrueOp, FalseOp, DAG.getConstant(C.CCValid, DL, MVT::i32),
2555 DAG.getConstant(C.CCMask, DL, MVT::i32), Glue};
2556
2557 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
2558}
2559
2560SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
2561 SelectionDAG &DAG) const {
2562 SDLoc DL(Node);
2563 const GlobalValue *GV = Node->getGlobal();
2564 int64_t Offset = Node->getOffset();
2565 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2566 CodeModel::Model CM = DAG.getTarget().getCodeModel();
2567
2568 SDValue Result;
2569 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
2570 // Assign anchors at 1<<12 byte boundaries.
2571 uint64_t Anchor = Offset & ~uint64_t(0xfff);
2572 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor);
2573 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2574
2575 // The offset can be folded into the address if it is aligned to a halfword.
2576 Offset -= Anchor;
2577 if (Offset != 0 && (Offset & 1) == 0) {
2578 SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
2579 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result);
2580 Offset = 0;
2581 }
2582 } else {
2583 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT);
2584 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2585 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
2586 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2587 }
2588
2589 // If there was a non-zero offset that we didn't fold, create an explicit
2590 // addition for it.
2591 if (Offset != 0)
2592 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
2593 DAG.getConstant(Offset, DL, PtrVT));
2594
2595 return Result;
2596}
2597
2598SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
2599 SelectionDAG &DAG,
2600 unsigned Opcode,
2601 SDValue GOTOffset) const {
2602 SDLoc DL(Node);
2603 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2604 SDValue Chain = DAG.getEntryNode();
2605 SDValue Glue;
2606
2607 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
2608 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
2609 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
2610 Glue = Chain.getValue(1);
2611 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
2612 Glue = Chain.getValue(1);
2613
2614 // The first call operand is the chain and the second is the TLS symbol.
2615 SmallVector<SDValue, 8> Ops;
2616 Ops.push_back(Chain);
2617 Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,
2618 Node->getValueType(0),
2619 0, 0));
2620
2621 // Add argument registers to the end of the list so that they are
2622 // known live into the call.
2623 Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT));
2624 Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT));
2625
2626 // Add a register mask operand representing the call-preserved registers.
2627 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2628 const uint32_t *Mask =
2629 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
2630 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 2630, __extension__ __PRETTY_FUNCTION__))
;
2631 Ops.push_back(DAG.getRegisterMask(Mask));
2632
2633 // Glue the call to the argument copies.
2634 Ops.push_back(Glue);
2635
2636 // Emit the call.
2637 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2638 Chain = DAG.getNode(Opcode, DL, NodeTys, Ops);
2639 Glue = Chain.getValue(1);
2640
2641 // Copy the return value from %r2.
2642 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
2643}
2644
2645SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
2646 SelectionDAG &DAG) const {
2647 SDValue Chain = DAG.getEntryNode();
2648 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2649
2650 // The high part of the thread pointer is in access register 0.
2651 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
2652 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
2653
2654 // The low part of the thread pointer is in access register 1.
2655 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
2656 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
2657
2658 // Merge them into a single 64-bit address.
2659 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
2660 DAG.getConstant(32, DL, PtrVT));
2661 return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
2662}
2663
2664SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
2665 SelectionDAG &DAG) const {
2666 if (DAG.getTarget().useEmulatedTLS())
2667 return LowerToTLSEmulatedModel(Node, DAG);
2668 SDLoc DL(Node);
2669 const GlobalValue *GV = Node->getGlobal();
2670 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2671 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
2672
2673 SDValue TP = lowerThreadPointer(DL, DAG);
2674
2675 // Get the offset of GA from the thread pointer, based on the TLS model.
2676 SDValue Offset;
2677 switch (model) {
2678 case TLSModel::GeneralDynamic: {
2679 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
2680 SystemZConstantPoolValue *CPV =
2681 SystemZConstantPoolValue::Create(GV, SystemZCP::TLSGD);
2682
2683 Offset = DAG.getConstantPool(CPV, PtrVT, 8);
2684 Offset = DAG.getLoad(
2685 PtrVT, DL, DAG.getEntryNode(), Offset,
2686 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2687
2688 // Call __tls_get_offset to retrieve the offset.
2689 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset);
2690 break;
2691 }
2692
2693 case TLSModel::LocalDynamic: {
2694 // Load the GOT offset of the module ID.
2695 SystemZConstantPoolValue *CPV =
2696 SystemZConstantPoolValue::Create(GV, SystemZCP::TLSLDM);
2697
2698 Offset = DAG.getConstantPool(CPV, PtrVT, 8);
2699 Offset = DAG.getLoad(
2700 PtrVT, DL, DAG.getEntryNode(), Offset,
2701 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2702
2703 // Call __tls_get_offset to retrieve the module base offset.
2704 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset);
2705
2706 // Note: The SystemZLDCleanupPass will remove redundant computations
2707 // of the module base offset. Count total number of local-dynamic
2708 // accesses to trigger execution of that pass.
2709 SystemZMachineFunctionInfo* MFI =
2710 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
2711 MFI->incNumLocalDynamicTLSAccesses();
2712
2713 // Add the per-symbol offset.
2714 CPV = SystemZConstantPoolValue::Create(GV, SystemZCP::DTPOFF);
2715
2716 SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, 8);
2717 DTPOffset = DAG.getLoad(
2718 PtrVT, DL, DAG.getEntryNode(), DTPOffset,
2719 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2720
2721 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset);
2722 break;
2723 }
2724
2725 case TLSModel::InitialExec: {
2726 // Load the offset from the GOT.
2727 Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
2728 SystemZII::MO_INDNTPOFF);
2729 Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset);
2730 Offset =
2731 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset,
2732 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2733 break;
2734 }
2735
2736 case TLSModel::LocalExec: {
2737 // Force the offset into the constant pool and load it from there.
2738 SystemZConstantPoolValue *CPV =
2739 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF);
2740
2741 Offset = DAG.getConstantPool(CPV, PtrVT, 8);
2742 Offset = DAG.getLoad(
2743 PtrVT, DL, DAG.getEntryNode(), Offset,
2744 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2745 break;
2746 }
2747 }
2748
2749 // Add the base and offset together.
2750 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset);
2751}
2752
2753SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
2754 SelectionDAG &DAG) const {
2755 SDLoc DL(Node);
2756 const BlockAddress *BA = Node->getBlockAddress();
2757 int64_t Offset = Node->getOffset();
2758 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2759
2760 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
2761 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2762 return Result;
2763}
2764
2765SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
2766 SelectionDAG &DAG) const {
2767 SDLoc DL(JT);
2768 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2769 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
2770
2771 // Use LARL to load the address of the table.
2772 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2773}
2774
2775SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
2776 SelectionDAG &DAG) const {
2777 SDLoc DL(CP);
2778 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2779
2780 SDValue Result;
2781 if (CP->isMachineConstantPoolEntry())
2782 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT,
2783 CP->getAlignment());
2784 else
2785 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT,
2786 CP->getAlignment(), CP->getOffset());
2787
2788 // Use LARL to load the address of the constant pool entry.
2789 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2790}
2791
2792SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
2793 SelectionDAG &DAG) const {
2794 MachineFunction &MF = DAG.getMachineFunction();
2795 MachineFrameInfo &MFI = MF.getFrameInfo();
2796 MFI.setFrameAddressIsTaken(true);
2797
2798 SDLoc DL(Op);
2799 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2800 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2801
2802 // If the back chain frame index has not been allocated yet, do so.
2803 SystemZMachineFunctionInfo *FI = MF.getInfo<SystemZMachineFunctionInfo>();
2804 int BackChainIdx = FI->getFramePointerSaveIndex();
2805 if (!BackChainIdx) {
2806 // By definition, the frame address is the address of the back chain.
2807 BackChainIdx = MFI.CreateFixedObject(8, -SystemZMC::CallFrameSize, false);
2808 FI->setFramePointerSaveIndex(BackChainIdx);
2809 }
2810 SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
2811
2812 // FIXME The frontend should detect this case.
2813 if (Depth > 0) {
2814 report_fatal_error("Unsupported stack frame traversal count");
2815 }
2816
2817 return BackChain;
2818}
2819
2820SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
2821 SelectionDAG &DAG) const {
2822 MachineFunction &MF = DAG.getMachineFunction();
2823 MachineFrameInfo &MFI = MF.getFrameInfo();
2824 MFI.setReturnAddressIsTaken(true);
2825
2826 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
2827 return SDValue();
2828
2829 SDLoc DL(Op);
2830 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2831 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2832
2833 // FIXME The frontend should detect this case.
2834 if (Depth > 0) {
2835 report_fatal_error("Unsupported stack frame traversal count");
2836 }
2837
2838 // Return R14D, which has the return address. Mark it an implicit live-in.
2839 unsigned LinkReg = MF.addLiveIn(SystemZ::R14D, &SystemZ::GR64BitRegClass);
2840 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT);
2841}
2842
2843SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
2844 SelectionDAG &DAG) const {
2845 SDLoc DL(Op);
2846 SDValue In = Op.getOperand(0);
2847 EVT InVT = In.getValueType();
2848 EVT ResVT = Op.getValueType();
2849
2850 // Convert loads directly. This is normally done by DAGCombiner,
2851 // but we need this case for bitcasts that are created during lowering
2852 // and which are then lowered themselves.
2853 if (auto *LoadN = dyn_cast<LoadSDNode>(In))
2854 if (ISD::isNormalLoad(LoadN)) {
2855 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
2856 LoadN->getBasePtr(), LoadN->getMemOperand());
2857 // Update the chain uses.
2858 DAG.ReplaceAllUsesOfValueWith(SDValue(LoadN, 1), NewLoad.getValue(1));
2859 return NewLoad;
2860 }
2861
2862 if (InVT == MVT::i32 && ResVT == MVT::f32) {
2863 SDValue In64;
2864 if (Subtarget.hasHighWord()) {
2865 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL,
2866 MVT::i64);
2867 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
2868 MVT::i64, SDValue(U64, 0), In);
2869 } else {
2870 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
2871 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
2872 DAG.getConstant(32, DL, MVT::i64));
2873 }
2874 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
2875 return DAG.getTargetExtractSubreg(SystemZ::subreg_r32,
2876 DL, MVT::f32, Out64);
2877 }
2878 if (InVT == MVT::f32 && ResVT == MVT::i32) {
2879 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
2880 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_r32, DL,
2881 MVT::f64, SDValue(U64, 0), In);
2882 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
2883 if (Subtarget.hasHighWord())
2884 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
2885 MVT::i32, Out64);
2886 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
2887 DAG.getConstant(32, DL, MVT::i64));
2888 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
2889 }
2890 llvm_unreachable("Unexpected bitcast combination")::llvm::llvm_unreachable_internal("Unexpected bitcast combination"
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 2890)
;
2891}
2892
2893SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
2894 SelectionDAG &DAG) const {
2895 MachineFunction &MF = DAG.getMachineFunction();
2896 SystemZMachineFunctionInfo *FuncInfo =
2897 MF.getInfo<SystemZMachineFunctionInfo>();
2898 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2899
2900 SDValue Chain = Op.getOperand(0);
2901 SDValue Addr = Op.getOperand(1);
2902 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
2903 SDLoc DL(Op);
2904
2905 // The initial values of each field.
2906 const unsigned NumFields = 4;
2907 SDValue Fields[NumFields] = {
2908 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
2909 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
2910 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),
2911 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)
2912 };
2913
2914 // Store each field into its respective slot.
2915 SDValue MemOps[NumFields];
2916 unsigned Offset = 0;
2917 for (unsigned I = 0; I < NumFields; ++I) {
2918 SDValue FieldAddr = Addr;
2919 if (Offset != 0)
2920 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
2921 DAG.getIntPtrConstant(Offset, DL));
2922 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
2923 MachinePointerInfo(SV, Offset));
2924 Offset += 8;
2925 }
2926 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
2927}
2928
2929SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
2930 SelectionDAG &DAG) const {
2931 SDValue Chain = Op.getOperand(0);
2932 SDValue DstPtr = Op.getOperand(1);
2933 SDValue SrcPtr = Op.getOperand(2);
2934 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
2935 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
2936 SDLoc DL(Op);
2937
2938 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32, DL),
2939 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false,
2940 /*isTailCall*/false,
2941 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
2942}
2943
2944SDValue SystemZTargetLowering::
2945lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
2946 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
2947 MachineFunction &MF = DAG.getMachineFunction();
2948 bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");
2949 bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
2950
2951 SDValue Chain = Op.getOperand(0);
2952 SDValue Size = Op.getOperand(1);
2953 SDValue Align = Op.getOperand(2);
2954 SDLoc DL(Op);
2955
2956 // If user has set the no alignment function attribute, ignore
2957 // alloca alignments.
2958 uint64_t AlignVal = (RealignOpt ?
2959 dyn_cast<ConstantSDNode>(Align)->getZExtValue() : 0);
2960
2961 uint64_t StackAlign = TFI->getStackAlignment();
2962 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
2963 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
2964
2965 unsigned SPReg = getStackPointerRegisterToSaveRestore();
2966 SDValue NeededSpace = Size;
2967
2968 // Get a reference to the stack pointer.
2969 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
2970
2971 // If we need a backchain, save it now.
2972 SDValue Backchain;
2973 if (StoreBackchain)
2974 Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo());
2975
2976 // Add extra space for alignment if needed.
2977 if (ExtraAlignSpace)
2978 NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace,
2979 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
2980
2981 // Get the new stack pointer value.
2982 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
2983
2984 // Copy the new stack pointer back.
2985 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP);
2986
2987 // The allocated data lives above the 160 bytes allocated for the standard
2988 // frame, plus any outgoing stack arguments. We don't know how much that
2989 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
2990 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
2991 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
2992
2993 // Dynamically realign if needed.
2994 if (RequiredAlign > StackAlign) {
2995 Result =
2996 DAG.getNode(ISD::ADD, DL, MVT::i64, Result,
2997 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
2998 Result =
2999 DAG.getNode(ISD::AND, DL, MVT::i64, Result,
3000 DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));
3001 }
3002
3003 if (StoreBackchain)
3004 Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo());
3005
3006 SDValue Ops[2] = { Result, Chain };
3007 return DAG.getMergeValues(Ops, DL);
3008}
3009
3010SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
3011 SDValue Op, SelectionDAG &DAG) const {
3012 SDLoc DL(Op);
3013
3014 return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
3015}
3016
3017SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
3018 SelectionDAG &DAG) const {
3019 EVT VT = Op.getValueType();
3020 SDLoc DL(Op);
3021 SDValue Ops[2];
3022 if (is32Bit(VT))
3023 // Just do a normal 64-bit multiplication and extract the results.
3024 // We define this so that it can be used for constant division.
3025 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0),
3026 Op.getOperand(1), Ops[1], Ops[0]);
3027 else if (Subtarget.hasMiscellaneousExtensions2())
3028 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
3029 // the high result in the even register. ISD::SMUL_LOHI is defined to
3030 // return the low half first, so the results are in reverse order.
3031 lowerGR128Binary(DAG, DL, VT, SystemZISD::SMUL_LOHI,
3032 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
3033 else {
3034 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
3035 //
3036 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
3037 //
3038 // but using the fact that the upper halves are either all zeros
3039 // or all ones:
3040 //
3041 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
3042 //
3043 // and grouping the right terms together since they are quicker than the
3044 // multiplication:
3045 //
3046 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
3047 SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
3048 SDValue LL = Op.getOperand(0);
3049 SDValue RL = Op.getOperand(1);
3050 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
3051 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
3052 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
3053 // the high result in the even register. ISD::SMUL_LOHI is defined to
3054 // return the low half first, so the results are in reverse order.
3055 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
3056 LL, RL, Ops[1], Ops[0]);
3057 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
3058 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
3059 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
3060 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum);
3061 }
3062 return DAG.getMergeValues(Ops, DL);
3063}
3064
3065SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
3066 SelectionDAG &DAG) const {
3067 EVT VT = Op.getValueType();
3068 SDLoc DL(Op);
3069 SDValue Ops[2];
3070 if (is32Bit(VT))
3071 // Just do a normal 64-bit multiplication and extract the results.
3072 // We define this so that it can be used for constant division.
3073 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0),
3074 Op.getOperand(1), Ops[1], Ops[0]);
3075 else
3076 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
3077 // the high result in the even register. ISD::UMUL_LOHI is defined to
3078 // return the low half first, so the results are in reverse order.
3079 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
3080 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
3081 return DAG.getMergeValues(Ops, DL);
3082}
3083
3084SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
3085 SelectionDAG &DAG) const {
3086 SDValue Op0 = Op.getOperand(0);
3087 SDValue Op1 = Op.getOperand(1);
3088 EVT VT = Op.getValueType();
3089 SDLoc DL(Op);
3090
3091 // We use DSGF for 32-bit division. This means the first operand must
3092 // always be 64-bit, and the second operand should be 32-bit whenever
3093 // that is possible, to improve performance.
3094 if (is32Bit(VT))
3095 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0);
3096 else if (DAG.ComputeNumSignBits(Op1) > 32)
3097 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
3098
3099 // DSG(F) returns the remainder in the even register and the
3100 // quotient in the odd register.
3101 SDValue Ops[2];
3102 lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]);
3103 return DAG.getMergeValues(Ops, DL);
3104}
3105
3106SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
3107 SelectionDAG &DAG) const {
3108 EVT VT = Op.getValueType();
3109 SDLoc DL(Op);
3110
3111 // DL(G) returns the remainder in the even register and the
3112 // quotient in the odd register.
3113 SDValue Ops[2];
3114 lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM,
3115 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
3116 return DAG.getMergeValues(Ops, DL);
3117}
3118
3119SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
3120 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation")(static_cast <bool> (Op.getValueType() == MVT::i64 &&
"Should be 64-bit operation") ? void (0) : __assert_fail ("Op.getValueType() == MVT::i64 && \"Should be 64-bit operation\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 3120, __extension__ __PRETTY_FUNCTION__))
;
3121
3122 // Get the known-zero masks for each operand.
3123 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) };
3124 KnownBits Known[2];
3125 DAG.computeKnownBits(Ops[0], Known[0]);
3126 DAG.computeKnownBits(Ops[1], Known[1]);
3127
3128 // See if the upper 32 bits of one operand and the lower 32 bits of the
3129 // other are known zero. They are the low and high operands respectively.
3130 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
3131 Known[1].Zero.getZExtValue() };
3132 unsigned High, Low;
3133 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
3134 High = 1, Low = 0;
3135 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
3136 High = 0, Low = 1;
3137 else
3138 return Op;
3139
3140 SDValue LowOp = Ops[Low];
3141 SDValue HighOp = Ops[High];
3142
3143 // If the high part is a constant, we're better off using IILH.
3144 if (HighOp.getOpcode() == ISD::Constant)
3145 return Op;
3146
3147 // If the low part is a constant that is outside the range of LHI,
3148 // then we're better off using IILF.
3149 if (LowOp.getOpcode() == ISD::Constant) {
3150 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue());
3151 if (!isInt<16>(Value))
3152 return Op;
3153 }
3154
3155 // Check whether the high part is an AND that doesn't change the
3156 // high 32 bits and just masks out low bits. We can skip it if so.
3157 if (HighOp.getOpcode() == ISD::AND &&
3158 HighOp.getOperand(1).getOpcode() == ISD::Constant) {
3159 SDValue HighOp0 = HighOp.getOperand(0);
3160 uint64_t Mask = cast<ConstantSDNode>(HighOp.getOperand(1))->getZExtValue();
3161 if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff))))
3162 HighOp = HighOp0;
3163 }
3164
3165 // Take advantage of the fact that all GR32 operations only change the
3166 // low 32 bits by truncating Low to an i32 and inserting it directly
3167 // using a subreg. The interesting cases are those where the truncation
3168 // can be folded.
3169 SDLoc DL(Op);
3170 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
3171 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
3172 MVT::i64, HighOp, Low32);
3173}
3174
3175SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
3176 SelectionDAG &DAG) const {
3177 EVT VT = Op.getValueType();
3178 SDLoc DL(Op);
3179 Op = Op.getOperand(0);
3180
3181 // Handle vector types via VPOPCT.
3182 if (VT.isVector()) {
3183 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
3184 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
3185 switch (VT.getScalarSizeInBits()) {
3186 case 8:
3187 break;
3188 case 16: {
3189 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
3190 SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
3191 SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
3192 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
3193 Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
3194 break;
3195 }
3196 case 32: {
3197 SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
3198 DAG.getConstant(0, DL, MVT::i32));
3199 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
3200 break;
3201 }
3202 case 64: {
3203 SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
3204 DAG.getConstant(0, DL, MVT::i32));
3205 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
3206 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
3207 break;
3208 }
3209 default:
3210 llvm_unreachable("Unexpected type")::llvm::llvm_unreachable_internal("Unexpected type", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 3210)
;
3211 }
3212 return Op;
3213 }
3214
3215 // Get the known-zero mask for the operand.
3216 KnownBits Known;
3217 DAG.computeKnownBits(Op, Known);
3218 unsigned NumSignificantBits = (~Known.Zero).getActiveBits();
3219 if (NumSignificantBits == 0)
3220 return DAG.getConstant(0, DL, VT);
3221
3222 // Skip known-zero high parts of the operand.
3223 int64_t OrigBitSize = VT.getSizeInBits();
3224 int64_t BitSize = (int64_t)1 << Log2_32_Ceil(NumSignificantBits);
3225 BitSize = std::min(BitSize, OrigBitSize);
3226
3227 // The POPCNT instruction counts the number of bits in each byte.
3228 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op);
3229 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op);
3230 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
3231
3232 // Add up per-byte counts in a binary tree. All bits of Op at
3233 // position larger than BitSize remain zero throughout.
3234 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
3235 SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
3236 if (BitSize != OrigBitSize)
3237 Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
3238 DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
3239 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
3240 }
3241
3242 // Extract overall result from high byte.
3243 if (BitSize > 8)
3244 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
3245 DAG.getConstant(BitSize - 8, DL, VT));
3246
3247 return Op;
3248}
3249
3250SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
3251 SelectionDAG &DAG) const {
3252 SDLoc DL(Op);
3253 AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
3254 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
3255 SyncScope::ID FenceSSID = static_cast<SyncScope::ID>(
3256 cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
3257
3258 // The only fence that needs an instruction is a sequentially-consistent
3259 // cross-thread fence.
3260 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
3261 FenceSSID == SyncScope::System) {
3262 return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
3263 Op.getOperand(0)),
3264 0);
3265 }
3266
3267 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
3268 return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
3269}
3270
3271// Op is an atomic load. Lower it into a normal volatile load.
3272SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
3273 SelectionDAG &DAG) const {
3274 auto *Node = cast<AtomicSDNode>(Op.getNode());
3275 return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
3276 Node->getChain(), Node->getBasePtr(),
3277 Node->getMemoryVT(), Node->getMemOperand());
3278}
3279
3280// Op is an atomic store. Lower it into a normal volatile store.
3281SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
3282 SelectionDAG &DAG) const {
3283 auto *Node = cast<AtomicSDNode>(Op.getNode());
3284 SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
3285 Node->getBasePtr(), Node->getMemoryVT(),
3286 Node->getMemOperand());
3287 // We have to enforce sequential consistency by performing a
3288 // serialization operation after the store.
3289 if (Node->getOrdering() == AtomicOrdering::SequentiallyConsistent)
3290 Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
3291 MVT::Other, Chain), 0);
3292 return Chain;
3293}
3294
3295// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
3296// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
3297SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
3298 SelectionDAG &DAG,
3299 unsigned Opcode) const {
3300 auto *Node = cast<AtomicSDNode>(Op.getNode());
3301
3302 // 32-bit operations need no code outside the main loop.
3303 EVT NarrowVT = Node->getMemoryVT();
3304 EVT WideVT = MVT::i32;
3305 if (NarrowVT == WideVT)
3306 return Op;
3307
3308 int64_t BitSize = NarrowVT.getSizeInBits();
3309 SDValue ChainIn = Node->getChain();
3310 SDValue Addr = Node->getBasePtr();
3311 SDValue Src2 = Node->getVal();
3312 MachineMemOperand *MMO = Node->getMemOperand();
3313 SDLoc DL(Node);
3314 EVT PtrVT = Addr.getValueType();
3315
3316 // Convert atomic subtracts of constants into additions.
3317 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
3318 if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
3319 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
3320 Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType());
3321 }
3322
3323 // Get the address of the containing word.
3324 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3325 DAG.getConstant(-4, DL, PtrVT));
3326
3327 // Get the number of bits that the word must be rotated left in order
3328 // to bring the field to the top bits of a GR32.
3329 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3330 DAG.getConstant(3, DL, PtrVT));
3331 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
3332
3333 // Get the complementing shift amount, for rotating a field in the top
3334 // bits back to its proper position.
3335 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3336 DAG.getConstant(0, DL, WideVT), BitShift);
3337
3338 // Extend the source operand to 32 bits and prepare it for the inner loop.
3339 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
3340 // operations require the source to be shifted in advance. (This shift
3341 // can be folded if the source is constant.) For AND and NAND, the lower
3342 // bits must be set, while for other opcodes they should be left clear.
3343 if (Opcode != SystemZISD::ATOMIC_SWAPW)
3344 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
3345 DAG.getConstant(32 - BitSize, DL, WideVT));
3346 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
3347 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
3348 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
3349 DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
3350
3351 // Construct the ATOMIC_LOADW_* node.
3352 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
3353 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
3354 DAG.getConstant(BitSize, DL, WideVT) };
3355 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
3356 NarrowVT, MMO);
3357
3358 // Rotate the result of the final CS so that the field is in the lower
3359 // bits of a GR32, then truncate it.
3360 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
3361 DAG.getConstant(BitSize, DL, WideVT));
3362 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
3363
3364 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
3365 return DAG.getMergeValues(RetOps, DL);
3366}
3367
3368// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations
3369// into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit
3370// operations into additions.
3371SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
3372 SelectionDAG &DAG) const {
3373 auto *Node = cast<AtomicSDNode>(Op.getNode());
3374 EVT MemVT = Node->getMemoryVT();
3375 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
3376 // A full-width operation.
3377 assert(Op.getValueType() == MemVT && "Mismatched VTs")(static_cast <bool> (Op.getValueType() == MemVT &&
"Mismatched VTs") ? void (0) : __assert_fail ("Op.getValueType() == MemVT && \"Mismatched VTs\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 3377, __extension__ __PRETTY_FUNCTION__))
;
3378 SDValue Src2 = Node->getVal();
3379 SDValue NegSrc2;
3380 SDLoc DL(Src2);
3381
3382 if (auto *Op2 = dyn_cast<ConstantSDNode>(Src2)) {
3383 // Use an addition if the operand is constant and either LAA(G) is
3384 // available or the negative value is in the range of A(G)FHI.
3385 int64_t Value = (-Op2->getAPIntValue()).getSExtValue();
3386 if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1())
3387 NegSrc2 = DAG.getConstant(Value, DL, MemVT);
3388 } else if (Subtarget.hasInterlockedAccess1())
3389 // Use LAA(G) if available.
3390 NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT),
3391 Src2);
3392
3393 if (NegSrc2.getNode())
3394 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
3395 Node->getChain(), Node->getBasePtr(), NegSrc2,
3396 Node->getMemOperand());
3397
3398 // Use the node as-is.
3399 return Op;
3400 }
3401
3402 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
3403}
3404
3405// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
3406SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
3407 SelectionDAG &DAG) const {
3408 auto *Node = cast<AtomicSDNode>(Op.getNode());
3409 SDValue ChainIn = Node->getOperand(0);
3410 SDValue Addr = Node->getOperand(1);
3411 SDValue CmpVal = Node->getOperand(2);
3412 SDValue SwapVal = Node->getOperand(3);
3413 MachineMemOperand *MMO = Node->getMemOperand();
3414 SDLoc DL(Node);
3415
3416 // We have native support for 32-bit and 64-bit compare and swap, but we
3417 // still need to expand extracting the "success" result from the CC.
3418 EVT NarrowVT = Node->getMemoryVT();
3419 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
3420 if (NarrowVT == WideVT) {
3421 SDVTList Tys = DAG.getVTList(WideVT, MVT::Other, MVT::Glue);
3422 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
3423 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP,
3424 DL, Tys, Ops, NarrowVT, MMO);
3425 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(2),
3426 SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
3427
3428 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
3429 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
3430 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(1));
3431 return SDValue();
3432 }
3433
3434 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
3435 // via a fullword ATOMIC_CMP_SWAPW operation.
3436 int64_t BitSize = NarrowVT.getSizeInBits();
3437 EVT PtrVT = Addr.getValueType();
3438
3439 // Get the address of the containing word.
3440 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3441 DAG.getConstant(-4, DL, PtrVT));
3442
3443 // Get the number of bits that the word must be rotated left in order
3444 // to bring the field to the top bits of a GR32.
3445 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3446 DAG.getConstant(3, DL, PtrVT));
3447 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
3448
3449 // Get the complementing shift amount, for rotating a field in the top
3450 // bits back to its proper position.
3451 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3452 DAG.getConstant(0, DL, WideVT), BitShift);
3453
3454 // Construct the ATOMIC_CMP_SWAPW node.
3455 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other, MVT::Glue);
3456 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
3457 NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
3458 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
3459 VTList, Ops, NarrowVT, MMO);
3460 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(2),
3461 SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
3462
3463 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
3464 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
3465 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(1));
3466 return SDValue();
3467}
3468
3469SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
3470 SelectionDAG &DAG) const {
3471 MachineFunction &MF = DAG.getMachineFunction();
3472 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true);
3473 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op),
3474 SystemZ::R15D, Op.getValueType());
3475}
3476
3477SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
3478 SelectionDAG &DAG) const {
3479 MachineFunction &MF = DAG.getMachineFunction();
3480 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true);
3481 bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
3482
3483 SDValue Chain = Op.getOperand(0);
3484 SDValue NewSP = Op.getOperand(1);
3485 SDValue Backchain;
3486 SDLoc DL(Op);
3487
3488 if (StoreBackchain) {
3489 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, MVT::i64);
3490 Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo());
3491 }
3492
3493 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R15D, NewSP);
3494
3495 if (StoreBackchain)
3496 Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo());
3497
3498 return Chain;
3499}
3500
3501SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
3502 SelectionDAG &DAG) const {
3503 bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
3504 if (!IsData)
3505 // Just preserve the chain.
3506 return Op.getOperand(0);
3507
3508 SDLoc DL(Op);
3509 bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
3510 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
3511 auto *Node = cast<MemIntrinsicSDNode>(Op.getNode());
3512 SDValue Ops[] = {
3513 Op.getOperand(0),
3514 DAG.getConstant(Code, DL, MVT::i32),
3515 Op.getOperand(1)
3516 };
3517 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
3518 Node->getVTList(), Ops,
3519 Node->getMemoryVT(), Node->getMemOperand());
3520}
3521
3522// Return an i32 that contains the value of CC immediately after After,
3523// whose final operand must be MVT::Glue.
3524static SDValue getCCResult(SelectionDAG &DAG, SDNode *After) {
3525 SDLoc DL(After);
3526 SDValue Glue = SDValue(After, After->getNumValues() - 1);
3527 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue);
3528 return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
3529 DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
3530}
3531
3532SDValue
3533SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
3534 SelectionDAG &DAG) const {
3535 unsigned Opcode, CCValid;
3536 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
3537 assert(Op->getNumValues() == 2 && "Expected only CC result and chain")(static_cast <bool> (Op->getNumValues() == 2 &&
"Expected only CC result and chain") ? void (0) : __assert_fail
("Op->getNumValues() == 2 && \"Expected only CC result and chain\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 3537, __extension__ __PRETTY_FUNCTION__))
;
3538 SDValue Glued = emitIntrinsicWithChainAndGlue(DAG, Op, Opcode);
3539 SDValue CC = getCCResult(DAG, Glued.getNode());
3540 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), CC);
3541 return SDValue();
3542 }
3543
3544 return SDValue();
3545}
3546
3547SDValue
3548SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
3549 SelectionDAG &DAG) const {
3550 unsigned Opcode, CCValid;
3551 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
3552 SDValue Glued = emitIntrinsicWithGlue(DAG, Op, Opcode);
3553 SDValue CC = getCCResult(DAG, Glued.getNode());
3554 if (Op->getNumValues() == 1)
3555 return CC;
3556 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result")(static_cast <bool> (Op->getNumValues() == 2 &&
"Expected a CC and non-CC result") ? void (0) : __assert_fail
("Op->getNumValues() == 2 && \"Expected a CC and non-CC result\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 3556, __extension__ __PRETTY_FUNCTION__))
;
3557 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(), Glued,
3558 CC);
3559 }
3560
3561 unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3562 switch (Id) {
3563 case Intrinsic::thread_pointer:
3564 return lowerThreadPointer(SDLoc(Op), DAG);
3565
3566 case Intrinsic::s390_vpdi:
3567 return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(),
3568 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3569
3570 case Intrinsic::s390_vperm:
3571 return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(),
3572 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
3573
3574 case Intrinsic::s390_vuphb:
3575 case Intrinsic::s390_vuphh:
3576 case Intrinsic::s390_vuphf:
3577 return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(),
3578 Op.getOperand(1));
3579
3580 case Intrinsic::s390_vuplhb:
3581 case Intrinsic::s390_vuplhh:
3582 case Intrinsic::s390_vuplhf:
3583 return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(),
3584 Op.getOperand(1));
3585
3586 case Intrinsic::s390_vuplb:
3587 case Intrinsic::s390_vuplhw:
3588 case Intrinsic::s390_vuplf:
3589 return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(),
3590 Op.getOperand(1));
3591
3592 case Intrinsic::s390_vupllb:
3593 case Intrinsic::s390_vupllh:
3594 case Intrinsic::s390_vupllf:
3595 return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(),
3596 Op.getOperand(1));
3597
3598 case Intrinsic::s390_vsumb:
3599 case Intrinsic::s390_vsumh:
3600 case Intrinsic::s390_vsumgh:
3601 case Intrinsic::s390_vsumgf:
3602 case Intrinsic::s390_vsumqf:
3603 case Intrinsic::s390_vsumqg:
3604 return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(),
3605 Op.getOperand(1), Op.getOperand(2));
3606 }
3607
3608 return SDValue();
3609}
3610
3611namespace {
3612// Says that SystemZISD operation Opcode can be used to perform the equivalent
3613// of a VPERM with permute vector Bytes. If Opcode takes three operands,
3614// Operand is the constant third operand, otherwise it is the number of
3615// bytes in each element of the result.
3616struct Permute {
3617 unsigned Opcode;
3618 unsigned Operand;
3619 unsigned char Bytes[SystemZ::VectorBytes];
3620};
3621}
3622
3623static const Permute PermuteForms[] = {
3624 // VMRHG
3625 { SystemZISD::MERGE_HIGH, 8,
3626 { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
3627 // VMRHF
3628 { SystemZISD::MERGE_HIGH, 4,
3629 { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
3630 // VMRHH
3631 { SystemZISD::MERGE_HIGH, 2,
3632 { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
3633 // VMRHB
3634 { SystemZISD::MERGE_HIGH, 1,
3635 { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
3636 // VMRLG
3637 { SystemZISD::MERGE_LOW, 8,
3638 { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
3639 // VMRLF
3640 { SystemZISD::MERGE_LOW, 4,
3641 { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
3642 // VMRLH
3643 { SystemZISD::MERGE_LOW, 2,
3644 { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
3645 // VMRLB
3646 { SystemZISD::MERGE_LOW, 1,
3647 { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
3648 // VPKG
3649 { SystemZISD::PACK, 4,
3650 { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
3651 // VPKF
3652 { SystemZISD::PACK, 2,
3653 { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
3654 // VPKH
3655 { SystemZISD::PACK, 1,
3656 { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
3657 // VPDI V1, V2, 4 (low half of V1, high half of V2)
3658 { SystemZISD::PERMUTE_DWORDS, 4,
3659 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
3660 // VPDI V1, V2, 1 (high half of V1, low half of V2)
3661 { SystemZISD::PERMUTE_DWORDS, 1,
3662 { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
3663};
3664
3665// Called after matching a vector shuffle against a particular pattern.
3666// Both the original shuffle and the pattern have two vector operands.
3667// OpNos[0] is the operand of the original shuffle that should be used for
3668// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
3669// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
3670// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
3671// for operands 0 and 1 of the pattern.
3672static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
3673 if (OpNos[0] < 0) {
3674 if (OpNos[1] < 0)
3675 return false;
3676 OpNo0 = OpNo1 = OpNos[1];
3677 } else if (OpNos[1] < 0) {
3678 OpNo0 = OpNo1 = OpNos[0];
3679 } else {
3680 OpNo0 = OpNos[0];
3681 OpNo1 = OpNos[1];
3682 }
3683 return true;
3684}
3685
3686// Bytes is a VPERM-like permute vector, except that -1 is used for
3687// undefined bytes. Return true if the VPERM can be implemented using P.
3688// When returning true set OpNo0 to the VPERM operand that should be
3689// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
3690//
3691// For example, if swapping the VPERM operands allows P to match, OpNo0
3692// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
3693// operand, but rewriting it to use two duplicated operands allows it to
3694// match P, then OpNo0 and OpNo1 will be the same.
3695static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
3696 unsigned &OpNo0, unsigned &OpNo1) {
3697 int OpNos[] = { -1, -1 };
3698 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
3699 int Elt = Bytes[I];
3700 if (Elt >= 0) {
3701 // Make sure that the two permute vectors use the same suboperand
3702 // byte number. Only the operand numbers (the high bits) are
3703 // allowed to differ.
3704 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
3705 return false;
3706 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
3707 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
3708 // Make sure that the operand mappings are consistent with previous
3709 // elements.
3710 if (OpNos[ModelOpNo] == 1 - RealOpNo)
3711 return false;
3712 OpNos[ModelOpNo] = RealOpNo;
3713 }
3714 }
3715 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
3716}
3717
3718// As above, but search for a matching permute.
3719static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
3720 unsigned &OpNo0, unsigned &OpNo1) {
3721 for (auto &P : PermuteForms)
3722 if (matchPermute(Bytes, P, OpNo0, OpNo1))
3723 return &P;
3724 return nullptr;
3725}
3726
3727// Bytes is a VPERM-like permute vector, except that -1 is used for
3728// undefined bytes. This permute is an operand of an outer permute.
3729// See whether redistributing the -1 bytes gives a shuffle that can be
3730// implemented using P. If so, set Transform to a VPERM-like permute vector
3731// that, when applied to the result of P, gives the original permute in Bytes.
3732static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
3733 const Permute &P,
3734 SmallVectorImpl<int> &Transform) {
3735 unsigned To = 0;
3736 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
3737 int Elt = Bytes[From];
3738 if (Elt < 0)
3739 // Byte number From of the result is undefined.
3740 Transform[From] = -1;
3741 else {
3742 while (P.Bytes[To] != Elt) {
3743 To += 1;
3744 if (To == SystemZ::VectorBytes)
3745 return false;
3746 }
3747 Transform[From] = To;
3748 }
3749 }
3750 return true;
3751}
3752
3753// As above, but search for a matching permute.
3754static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
3755 SmallVectorImpl<int> &Transform) {
3756 for (auto &P : PermuteForms)
3757 if (matchDoublePermute(Bytes, P, Transform))
3758 return &P;
3759 return nullptr;
3760}
3761
3762// Convert the mask of the given VECTOR_SHUFFLE into a byte-level mask,
3763// as if it had type vNi8.
3764static void getVPermMask(ShuffleVectorSDNode *VSN,
3765 SmallVectorImpl<int> &Bytes) {
3766 EVT VT = VSN->getValueType(0);
3767 unsigned NumElements = VT.getVectorNumElements();
3768 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
3769 Bytes.resize(NumElements * BytesPerElement, -1);
3770 for (unsigned I = 0; I < NumElements; ++I) {
3771 int Index = VSN->getMaskElt(I);
3772 if (Index >= 0)
3773 for (unsigned J = 0; J < BytesPerElement; ++J)
3774 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
3775 }
3776}
3777
3778// Bytes is a VPERM-like permute vector, except that -1 is used for
3779// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
3780// the result come from a contiguous sequence of bytes from one input.
3781// Set Base to the selector for the first byte if so.
3782static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
3783 unsigned BytesPerElement, int &Base) {
3784 Base = -1;
3785 for (unsigned I = 0; I < BytesPerElement; ++I) {
3786 if (Bytes[Start + I] >= 0) {
3787 unsigned Elem = Bytes[Start + I];
3788 if (Base < 0) {
3789 Base = Elem - I;
3790 // Make sure the bytes would come from one input operand.
3791 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
3792 return false;
3793 } else if (unsigned(Base) != Elem - I)
3794 return false;
3795 }
3796 }
3797 return true;
3798}
3799
3800// Bytes is a VPERM-like permute vector, except that -1 is used for
3801// undefined bytes. Return true if it can be performed using VSLDI.
3802// When returning true, set StartIndex to the shift amount and OpNo0
3803// and OpNo1 to the VPERM operands that should be used as the first
3804// and second shift operand respectively.
3805static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
3806 unsigned &StartIndex, unsigned &OpNo0,
3807 unsigned &OpNo1) {
3808 int OpNos[] = { -1, -1 };
3809 int Shift = -1;
3810 for (unsigned I = 0; I < 16; ++I) {
3811 int Index = Bytes[I];
3812 if (Index >= 0) {
3813 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
3814 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
3815 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
3816 if (Shift < 0)
3817 Shift = ExpectedShift;
3818 else if (Shift != ExpectedShift)
3819 return false;
3820 // Make sure that the operand mappings are consistent with previous
3821 // elements.
3822 if (OpNos[ModelOpNo] == 1 - RealOpNo)
3823 return false;
3824 OpNos[ModelOpNo] = RealOpNo;
3825 }
3826 }
3827 StartIndex = Shift;
3828 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
3829}
3830
3831// Create a node that performs P on operands Op0 and Op1, casting the
3832// operands to the appropriate type. The type of the result is determined by P.
3833static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
3834 const Permute &P, SDValue Op0, SDValue Op1) {
3835 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
3836 // elements of a PACK are twice as wide as the outputs.
3837 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
3838 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
3839 P.Operand);
3840 // Cast both operands to the appropriate type.
3841 MVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBytes * 8),
3842 SystemZ::VectorBytes / InBytes);
3843 Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
3844 Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
3845 SDValue Op;
3846 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
3847 SDValue Op2 = DAG.getConstant(P.Operand, DL, MVT::i32);
3848 Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
3849 } else if (P.Opcode == SystemZISD::PACK) {
3850 MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8),
3851 SystemZ::VectorBytes / P.Operand);
3852 Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
3853 } else {
3854 Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
3855 }
3856 return Op;
3857}
3858
3859// Bytes is a VPERM-like permute vector, except that -1 is used for
3860// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
3861// VSLDI or VPERM.
3862static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
3863 SDValue *Ops,
3864 const SmallVectorImpl<int> &Bytes) {
3865 for (unsigned I = 0; I < 2; ++I)
3866 Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
3867
3868 // First see whether VSLDI can be used.
3869 unsigned StartIndex, OpNo0, OpNo1;
3870 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
3871 return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
3872 Ops[OpNo1], DAG.getConstant(StartIndex, DL, MVT::i32));
3873
3874 // Fall back on VPERM. Construct an SDNode for the permute vector.
3875 SDValue IndexNodes[SystemZ::VectorBytes];
3876 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
3877 if (Bytes[I] >= 0)
3878 IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
3879 else
3880 IndexNodes[I] = DAG.getUNDEF(MVT::i32);
3881 SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
3882 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0], Ops[1], Op2);
3883}
3884
3885namespace {
3886// Describes a general N-operand vector shuffle.
3887struct GeneralShuffle {
3888 GeneralShuffle(EVT vt) : VT(vt) {}
3889 void addUndef();
3890 bool add(SDValue, unsigned);
3891 SDValue getNode(SelectionDAG &, const SDLoc &);
3892
3893 // The operands of the shuffle.
3894 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
3895
3896 // Index I is -1 if byte I of the result is undefined. Otherwise the
3897 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
3898 // Bytes[I] / SystemZ::VectorBytes.
3899 SmallVector<int, SystemZ::VectorBytes> Bytes;
3900
3901 // The type of the shuffle result.
3902 EVT VT;
3903};
3904}
3905
3906// Add an extra undefined element to the shuffle.
3907void GeneralShuffle::addUndef() {
3908 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
3909 for (unsigned I = 0; I < BytesPerElement; ++I)
3910 Bytes.push_back(-1);
3911}
3912
3913// Add an extra element to the shuffle, taking it from element Elem of Op.
3914// A null Op indicates a vector input whose value will be calculated later;
3915// there is at most one such input per shuffle and it always has the same
3916// type as the result. Aborts and returns false if the source vector elements
3917// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
3918// LLVM they become implicitly extended, but this is rare and not optimized.
3919bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
3920 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
3921
3922 // The source vector can have wider elements than the result,
3923 // either through an explicit TRUNCATE or because of type legalization.
3924 // We want the least significant part.
3925 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
3926 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
3927
3928 // Return false if the source elements are smaller than their destination
3929 // elements.
3930 if (FromBytesPerElement < BytesPerElement)
3931 return false;
3932
3933 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
3934 (FromBytesPerElement - BytesPerElement));
3935
3936 // Look through things like shuffles and bitcasts.
3937 while (Op.getNode()) {
3938 if (Op.getOpcode() == ISD::BITCAST)
3939 Op = Op.getOperand(0);
3940 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
3941 // See whether the bytes we need come from a contiguous part of one
3942 // operand.
3943 SmallVector<int, SystemZ::VectorBytes> OpBytes;
3944 getVPermMask(cast<ShuffleVectorSDNode>(Op), OpBytes);
3945 int NewByte;
3946 if (!getShuffleInput(OpBytes, Byte, BytesPerElement, NewByte))
3947 break;
3948 if (NewByte < 0) {
3949 addUndef();
3950 return true;
3951 }
3952 Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes);
3953 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
3954 } else if (Op.isUndef()) {
3955 addUndef();
3956 return true;
3957 } else
3958 break;
3959 }
3960
3961 // Make sure that the source of the extraction is in Ops.
3962 unsigned OpNo = 0;
3963 for (; OpNo < Ops.size(); ++OpNo)
3964 if (Ops[OpNo] == Op)
3965 break;
3966 if (OpNo == Ops.size())
3967 Ops.push_back(Op);
3968
3969 // Add the element to Bytes.
3970 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
3971 for (unsigned I = 0; I < BytesPerElement; ++I)
3972 Bytes.push_back(Base + I);
3973
3974 return true;
3975}
3976
3977// Return SDNodes for the completed shuffle.
3978SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
3979 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector")(static_cast <bool> (Bytes.size() == SystemZ::VectorBytes
&& "Incomplete vector") ? void (0) : __assert_fail (
"Bytes.size() == SystemZ::VectorBytes && \"Incomplete vector\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 3979, __extension__ __PRETTY_FUNCTION__))
;
3980
3981 if (Ops.size() == 0)
3982 return DAG.getUNDEF(VT);
3983
3984 // Make sure that there are at least two shuffle operands.
3985 if (Ops.size() == 1)
3986 Ops.push_back(DAG.getUNDEF(MVT::v16i8));
3987
3988 // Create a tree of shuffles, deferring root node until after the loop.
3989 // Try to redistribute the undefined elements of non-root nodes so that
3990 // the non-root shuffles match something like a pack or merge, then adjust
3991 // the parent node's permute vector to compensate for the new order.
3992 // Among other things, this copes with vectors like <2 x i16> that were
3993 // padded with undefined elements during type legalization.
3994 //
3995 // In the best case this redistribution will lead to the whole tree
3996 // using packs and merges. It should rarely be a loss in other cases.
3997 unsigned Stride = 1;
3998 for (; Stride * 2 < Ops.size(); Stride *= 2) {
3999 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
4000 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
4001
4002 // Create a mask for just these two operands.
4003 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
4004 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
4005 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
4006 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
4007 if (OpNo == I)
4008 NewBytes[J] = Byte;
4009 else if (OpNo == I + Stride)
4010 NewBytes[J] = SystemZ::VectorBytes + Byte;
4011 else
4012 NewBytes[J] = -1;
4013 }
4014 // See if it would be better to reorganize NewMask to avoid using VPERM.
4015 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
4016 if (const Permute *P = matchDoublePermute(NewBytes, NewBytesMap)) {
4017 Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
4018 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
4019 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
4020 if (NewBytes[J] >= 0) {
4021 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&(static_cast <bool> (unsigned(NewBytesMap[J]) < SystemZ
::VectorBytes && "Invalid double permute") ? void (0)
: __assert_fail ("unsigned(NewBytesMap[J]) < SystemZ::VectorBytes && \"Invalid double permute\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4022, __extension__ __PRETTY_FUNCTION__))
4022 "Invalid double permute")(static_cast <bool> (unsigned(NewBytesMap[J]) < SystemZ
::VectorBytes && "Invalid double permute") ? void (0)
: __assert_fail ("unsigned(NewBytesMap[J]) < SystemZ::VectorBytes && \"Invalid double permute\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4022, __extension__ __PRETTY_FUNCTION__))
;
4023 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
4024 } else
4025 assert(NewBytesMap[J] < 0 && "Invalid double permute")(static_cast <bool> (NewBytesMap[J] < 0 && "Invalid double permute"
) ? void (0) : __assert_fail ("NewBytesMap[J] < 0 && \"Invalid double permute\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4025, __extension__ __PRETTY_FUNCTION__))
;
4026 }
4027 } else {
4028 // Just use NewBytes on the operands.
4029 Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
4030 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
4031 if (NewBytes[J] >= 0)
4032 Bytes[J] = I * SystemZ::VectorBytes + J;
4033 }
4034 }
4035 }
4036
4037 // Now we just have 2 inputs. Put the second operand in Ops[1].
4038 if (Stride > 1) {
4039 Ops[1] = Ops[Stride];
4040 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
4041 if (Bytes[I] >= int(SystemZ::VectorBytes))
4042 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
4043 }
4044
4045 // Look for an instruction that can do the permute without resorting
4046 // to VPERM.
4047 unsigned OpNo0, OpNo1;
4048 SDValue Op;
4049 if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
4050 Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
4051 else
4052 Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
4053 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4054}
4055
4056// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
4057static bool isScalarToVector(SDValue Op) {
4058 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
4059 if (!Op.getOperand(I).isUndef())
4060 return false;
4061 return true;
4062}
4063
4064// Return a vector of type VT that contains Value in the first element.
4065// The other elements don't matter.
4066static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4067 SDValue Value) {
4068 // If we have a constant, replicate it to all elements and let the
4069 // BUILD_VECTOR lowering take care of it.
4070 if (Value.getOpcode() == ISD::Constant ||
4071 Value.getOpcode() == ISD::ConstantFP) {
4072 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
4073 return DAG.getBuildVector(VT, DL, Ops);
4074 }
4075 if (Value.isUndef())
4076 return DAG.getUNDEF(VT);
4077 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
4078}
4079
4080// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
4081// element 1. Used for cases in which replication is cheap.
4082static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4083 SDValue Op0, SDValue Op1) {
4084 if (Op0.isUndef()) {
4085 if (Op1.isUndef())
4086 return DAG.getUNDEF(VT);
4087 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
4088 }
4089 if (Op1.isUndef())
4090 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
4091 return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
4092 buildScalarToVector(DAG, DL, VT, Op0),
4093 buildScalarToVector(DAG, DL, VT, Op1));
4094}
4095
4096// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
4097// vector for them.
4098static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
4099 SDValue Op1) {
4100 if (Op0.isUndef() && Op1.isUndef())
4101 return DAG.getUNDEF(MVT::v2i64);
4102 // If one of the two inputs is undefined then replicate the other one,
4103 // in order to avoid using another register unnecessarily.
4104 if (Op0.isUndef())
4105 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
4106 else if (Op1.isUndef())
4107 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
4108 else {
4109 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
4110 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
4111 }
4112 return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
4113}
4114
4115// Try to represent constant BUILD_VECTOR node BVN using a
4116// SystemZISD::BYTE_MASK-style mask. Store the mask value in Mask
4117// on success.
4118static bool tryBuildVectorByteMask(BuildVectorSDNode *BVN, uint64_t &Mask) {
4119 EVT ElemVT = BVN->getValueType(0).getVectorElementType();
4120 unsigned BytesPerElement = ElemVT.getStoreSize();
4121 for (unsigned I = 0, E = BVN->getNumOperands(); I != E; ++I) {
4122 SDValue Op = BVN->getOperand(I);
4123 if (!Op.isUndef()) {
4124 uint64_t Value;
4125 if (Op.getOpcode() == ISD::Constant)
4126 Value = dyn_cast<ConstantSDNode>(Op)->getZExtValue();
4127 else if (Op.getOpcode() == ISD::ConstantFP)
4128 Value = (dyn_cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt()
4129 .getZExtValue());
4130 else
4131 return false;
4132 for (unsigned J = 0; J < BytesPerElement; ++J) {
4133 uint64_t Byte = (Value >> (J * 8)) & 0xff;
4134 if (Byte == 0xff)
4135 Mask |= 1ULL << ((E - I - 1) * BytesPerElement + J);
4136 else if (Byte != 0)
4137 return false;
4138 }
4139 }
4140 }
4141 return true;
4142}
4143
4144// Try to load a vector constant in which BitsPerElement-bit value Value
4145// is replicated to fill the vector. VT is the type of the resulting
4146// constant, which may have elements of a different size from BitsPerElement.
4147// Return the SDValue of the constant on success, otherwise return
4148// an empty value.
4149static SDValue tryBuildVectorReplicate(SelectionDAG &DAG,
4150 const SystemZInstrInfo *TII,
4151 const SDLoc &DL, EVT VT, uint64_t Value,
4152 unsigned BitsPerElement) {
4153 // Signed 16-bit values can be replicated using VREPI.
4154 int64_t SignedValue = SignExtend64(Value, BitsPerElement);
4155 if (isInt<16>(SignedValue)) {
4156 MVT VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement),
4157 SystemZ::VectorBits / BitsPerElement);
4158 SDValue Op = DAG.getNode(SystemZISD::REPLICATE, DL, VecVT,
4159 DAG.getConstant(SignedValue, DL, MVT::i32));
4160 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4161 }
4162 // See whether rotating the constant left some N places gives a value that
4163 // is one less than a power of 2 (i.e. all zeros followed by all ones).
4164 // If so we can use VGM.
4165 unsigned Start, End;
4166 if (TII->isRxSBGMask(Value, BitsPerElement, Start, End)) {
4167 // isRxSBGMask returns the bit numbers for a full 64-bit value,
4168 // with 0 denoting 1 << 63 and 63 denoting 1. Convert them to
4169 // bit numbers for an BitsPerElement value, so that 0 denotes
4170 // 1 << (BitsPerElement-1).
4171 Start -= 64 - BitsPerElement;
4172 End -= 64 - BitsPerElement;
4173 MVT VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement),
4174 SystemZ::VectorBits / BitsPerElement);
4175 SDValue Op = DAG.getNode(SystemZISD::ROTATE_MASK, DL, VecVT,
4176 DAG.getConstant(Start, DL, MVT::i32),
4177 DAG.getConstant(End, DL, MVT::i32));
4178 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4179 }
4180 return SDValue();
4181}
4182
4183// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
4184// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
4185// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
4186// would benefit from this representation and return it if so.
4187static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
4188 BuildVectorSDNode *BVN) {
4189 EVT VT = BVN->getValueType(0);
4190 unsigned NumElements = VT.getVectorNumElements();
4191
4192 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
4193 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
4194 // need a BUILD_VECTOR, add an additional placeholder operand for that
4195 // BUILD_VECTOR and store its operands in ResidueOps.
4196 GeneralShuffle GS(VT);
4197 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
4198 bool FoundOne = false;
4199 for (unsigned I = 0; I < NumElements; ++I) {
4200 SDValue Op = BVN->getOperand(I);
4201 if (Op.getOpcode() == ISD::TRUNCATE)
4202 Op = Op.getOperand(0);
4203 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4204 Op.getOperand(1).getOpcode() == ISD::Constant) {
4205 unsigned Elem = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4206 if (!GS.add(Op.getOperand(0), Elem))
4207 return SDValue();
4208 FoundOne = true;
4209 } else if (Op.isUndef()) {
4210 GS.addUndef();
4211 } else {
4212 if (!GS.add(SDValue(), ResidueOps.size()))
4213 return SDValue();
4214 ResidueOps.push_back(BVN->getOperand(I));
4215 }
4216 }
4217
4218 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
4219 if (!FoundOne)
4220 return SDValue();
4221
4222 // Create the BUILD_VECTOR for the remaining elements, if any.
4223 if (!ResidueOps.empty()) {
4224 while (ResidueOps.size() < NumElements)
4225 ResidueOps.push_back(DAG.getUNDEF(ResidueOps[0].getValueType()));
4226 for (auto &Op : GS.Ops) {
4227 if (!Op.getNode()) {
4228 Op = DAG.getBuildVector(VT, SDLoc(BVN), ResidueOps);
4229 break;
4230 }
4231 }
4232 }
4233 return GS.getNode(DAG, SDLoc(BVN));
4234}
4235
4236// Combine GPR scalar values Elems into a vector of type VT.
4237static SDValue buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4238 SmallVectorImpl<SDValue> &Elems) {
4239 // See whether there is a single replicated value.
4240 SDValue Single;
4241 unsigned int NumElements = Elems.size();
4242 unsigned int Count = 0;
4243 for (auto Elem : Elems) {
4244 if (!Elem.isUndef()) {
4245 if (!Single.getNode())
4246 Single = Elem;
4247 else if (Elem != Single) {
4248 Single = SDValue();
4249 break;
4250 }
4251 Count += 1;
4252 }
4253 }
4254 // There are three cases here:
4255 //
4256 // - if the only defined element is a loaded one, the best sequence
4257 // is a replicating load.
4258 //
4259 // - otherwise, if the only defined element is an i64 value, we will
4260 // end up with the same VLVGP sequence regardless of whether we short-cut
4261 // for replication or fall through to the later code.
4262 //
4263 // - otherwise, if the only defined element is an i32 or smaller value,
4264 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
4265 // This is only a win if the single defined element is used more than once.
4266 // In other cases we're better off using a single VLVGx.
4267 if (Single.getNode() && (Count > 1 || Single.getOpcode() == ISD::LOAD))
4268 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
4269
4270 // If all elements are loads, use VLREP/VLEs (below).
4271 bool AllLoads = true;
4272 for (auto Elem : Elems)
4273 if (Elem.getOpcode() != ISD::LOAD || cast<LoadSDNode>(Elem)->isIndexed()) {
4274 AllLoads = false;
4275 break;
4276 }
4277
4278 // The best way of building a v2i64 from two i64s is to use VLVGP.
4279 if (VT == MVT::v2i64 && !AllLoads)
4280 return joinDwords(DAG, DL, Elems[0], Elems[1]);
4281
4282 // Use a 64-bit merge high to combine two doubles.
4283 if (VT == MVT::v2f64 && !AllLoads)
4284 return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
4285
4286 // Build v4f32 values directly from the FPRs:
4287 //
4288 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
4289 // V V VMRHF
4290 // <ABxx> <CDxx>
4291 // V VMRHG
4292 // <ABCD>
4293 if (VT == MVT::v4f32 && !AllLoads) {
4294 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
4295 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
4296 // Avoid unnecessary undefs by reusing the other operand.
4297 if (Op01.isUndef())
4298 Op01 = Op23;
4299 else if (Op23.isUndef())
4300 Op23 = Op01;
4301 // Merging identical replications is a no-op.
4302 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
4303 return Op01;
4304 Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01);
4305 Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23);
4306 SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH,
4307 DL, MVT::v2i64, Op01, Op23);
4308 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4309 }
4310
4311 // Collect the constant terms.
4312 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
4313 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
4314
4315 unsigned NumConstants = 0;
4316 for (unsigned I = 0; I < NumElements; ++I) {
4317 SDValue Elem = Elems[I];
4318 if (Elem.getOpcode() == ISD::Constant ||
4319 Elem.getOpcode() == ISD::ConstantFP) {
4320 NumConstants += 1;
4321 Constants[I] = Elem;
4322 Done[I] = true;
4323 }
4324 }
4325 // If there was at least one constant, fill in the other elements of
4326 // Constants with undefs to get a full vector constant and use that
4327 // as the starting point.
4328 SDValue Result;
4329 if (NumConstants > 0) {
4330 for (unsigned I = 0; I < NumElements; ++I)
4331 if (!Constants[I].getNode())
4332 Constants[I] = DAG.getUNDEF(Elems[I].getValueType());
4333 Result = DAG.getBuildVector(VT, DL, Constants);
4334 } else {
4335 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
4336 // avoid a false dependency on any previous contents of the vector
4337 // register.
4338
4339 // Use a VLREP if at least one element is a load.
4340 unsigned LoadElIdx = UINT_MAX(2147483647 *2U +1U);
4341 for (unsigned I = 0; I < NumElements; ++I)
4342 if (Elems[I].getOpcode() == ISD::LOAD &&
4343 cast<LoadSDNode>(Elems[I])->isUnindexed()) {
4344 LoadElIdx = I;
4345 break;
4346 }
4347 if (LoadElIdx != UINT_MAX(2147483647 *2U +1U)) {
4348 Result = DAG.getNode(SystemZISD::REPLICATE, DL, VT, Elems[LoadElIdx]);
4349 Done[LoadElIdx] = true;
4350 } else {
4351 // Try to use VLVGP.
4352 unsigned I1 = NumElements / 2 - 1;
4353 unsigned I2 = NumElements - 1;
4354 bool Def1 = !Elems[I1].isUndef();
4355 bool Def2 = !Elems[I2].isUndef();
4356 if (Def1 || Def2) {
4357 SDValue Elem1 = Elems[Def1 ? I1 : I2];
4358 SDValue Elem2 = Elems[Def2 ? I2 : I1];
4359 Result = DAG.getNode(ISD::BITCAST, DL, VT,
4360 joinDwords(DAG, DL, Elem1, Elem2));
4361 Done[I1] = true;
4362 Done[I2] = true;
4363 } else
4364 Result = DAG.getUNDEF(VT);
4365 }
4366 }
4367
4368 // Use VLVGx to insert the other elements.
4369 for (unsigned I = 0; I < NumElements; ++I)
4370 if (!Done[I] && !Elems[I].isUndef())
4371 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I],
4372 DAG.getConstant(I, DL, MVT::i32));
4373 return Result;
4374}
4375
4376SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
4377 SelectionDAG &DAG) const {
4378 const SystemZInstrInfo *TII =
4379 static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
4380 auto *BVN = cast<BuildVectorSDNode>(Op.getNode());
4381 SDLoc DL(Op);
4382 EVT VT = Op.getValueType();
4383
4384 if (BVN->isConstant()) {
3
Assuming the condition is true
4
Taking true branch
4385 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
4386 // preferred way of creating all-zero and all-one vectors so give it
4387 // priority over other methods below.
4388 uint64_t Mask = 0;
4389 if (tryBuildVectorByteMask(BVN, Mask)) {
5
Taking false branch
4390 SDValue Op = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
4391 DAG.getConstant(Mask, DL, MVT::i32));
4392 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4393 }
4394
4395 // Try using some form of replication.
4396 APInt SplatBits, SplatUndef;
4397 unsigned SplatBitSize;
4398 bool HasAnyUndefs;
4399 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6
Assuming the condition is true
8
Taking true branch
4400 8, true) &&
4401 SplatBitSize <= 64) {
7
Assuming 'SplatBitSize' is <= 64
4402 // First try assuming that any undefined bits above the highest set bit
4403 // and below the lowest set bit are 1s. This increases the likelihood of
4404 // being able to use a sign-extended element value in VECTOR REPLICATE
4405 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
4406 uint64_t SplatBitsZ = SplatBits.getZExtValue();
4407 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
4408 uint64_t Lower = (SplatUndefZ
4409 & ((uint64_t(1) << findFirstSet(SplatBitsZ)) - 1));
9
The result of the left shift is undefined due to shifting by '18446744073709551615', which is greater or equal to the width of type 'uint64_t'
4410 uint64_t Upper = (SplatUndefZ
4411 & ~((uint64_t(1) << findLastSet(SplatBitsZ)) - 1));
4412 uint64_t Value = SplatBitsZ | Upper | Lower;
4413 SDValue Op = tryBuildVectorReplicate(DAG, TII, DL, VT, Value,
4414 SplatBitSize);
4415 if (Op.getNode())
4416 return Op;
4417
4418 // Now try assuming that any undefined bits between the first and
4419 // last defined set bits are set. This increases the chances of
4420 // using a non-wraparound mask.
4421 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
4422 Value = SplatBitsZ | Middle;
4423 Op = tryBuildVectorReplicate(DAG, TII, DL, VT, Value, SplatBitSize);
4424 if (Op.getNode())
4425 return Op;
4426 }
4427
4428 // Fall back to loading it from memory.
4429 return SDValue();
4430 }
4431
4432 // See if we should use shuffles to construct the vector from other vectors.
4433 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
4434 return Res;
4435
4436 // Detect SCALAR_TO_VECTOR conversions.
4437 if (isOperationLegal(ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
4438 return buildScalarToVector(DAG, DL, VT, Op.getOperand(0));
4439
4440 // Otherwise use buildVector to build the vector up from GPRs.
4441 unsigned NumElements = Op.getNumOperands();
4442 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
4443 for (unsigned I = 0; I < NumElements; ++I)
4444 Ops[I] = Op.getOperand(I);
4445 return buildVector(DAG, DL, VT, Ops);
4446}
4447
4448SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
4449 SelectionDAG &DAG) const {
4450 auto *VSN = cast<ShuffleVectorSDNode>(Op.getNode());
4451 SDLoc DL(Op);
4452 EVT VT = Op.getValueType();
4453 unsigned NumElements = VT.getVectorNumElements();
4454
4455 if (VSN->isSplat()) {
4456 SDValue Op0 = Op.getOperand(0);
4457 unsigned Index = VSN->getSplatIndex();
4458 assert(Index < VT.getVectorNumElements() &&(static_cast <bool> (Index < VT.getVectorNumElements
() && "Splat index should be defined and in first operand"
) ? void (0) : __assert_fail ("Index < VT.getVectorNumElements() && \"Splat index should be defined and in first operand\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4459, __extension__ __PRETTY_FUNCTION__))
4459 "Splat index should be defined and in first operand")(static_cast <bool> (Index < VT.getVectorNumElements
() && "Splat index should be defined and in first operand"
) ? void (0) : __assert_fail ("Index < VT.getVectorNumElements() && \"Splat index should be defined and in first operand\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4459, __extension__ __PRETTY_FUNCTION__))
;
4460 // See whether the value we're splatting is directly available as a scalar.
4461 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
4462 Op0.getOpcode() == ISD::BUILD_VECTOR)
4463 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
4464 // Otherwise keep it as a vector-to-vector operation.
4465 return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
4466 DAG.getConstant(Index, DL, MVT::i32));
4467 }
4468
4469 GeneralShuffle GS(VT);
4470 for (unsigned I = 0; I < NumElements; ++I) {
4471 int Elt = VSN->getMaskElt(I);
4472 if (Elt < 0)
4473 GS.addUndef();
4474 else if (!GS.add(Op.getOperand(unsigned(Elt) / NumElements),
4475 unsigned(Elt) % NumElements))
4476 return SDValue();
4477 }
4478 return GS.getNode(DAG, SDLoc(VSN));
4479}
4480
4481SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
4482 SelectionDAG &DAG) const {
4483 SDLoc DL(Op);
4484 // Just insert the scalar into element 0 of an undefined vector.
4485 return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL,
4486 Op.getValueType(), DAG.getUNDEF(Op.getValueType()),
4487 Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32));
4488}
4489
4490SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4491 SelectionDAG &DAG) const {
4492 // Handle insertions of floating-point values.
4493 SDLoc DL(Op);
4494 SDValue Op0 = Op.getOperand(0);
4495 SDValue Op1 = Op.getOperand(1);
4496 SDValue Op2 = Op.getOperand(2);
4497 EVT VT = Op.getValueType();
4498
4499 // Insertions into constant indices of a v2f64 can be done using VPDI.
4500 // However, if the inserted value is a bitcast or a constant then it's
4501 // better to use GPRs, as below.
4502 if (VT == MVT::v2f64 &&
4503 Op1.getOpcode() != ISD::BITCAST &&
4504 Op1.getOpcode() != ISD::ConstantFP &&
4505 Op2.getOpcode() == ISD::Constant) {
4506 uint64_t Index = dyn_cast<ConstantSDNode>(Op2)->getZExtValue();
4507 unsigned Mask = VT.getVectorNumElements() - 1;
4508 if (Index <= Mask)
4509 return Op;
4510 }
4511
4512 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
4513 MVT IntVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
4514 MVT IntVecVT = MVT::getVectorVT(IntVT, VT.getVectorNumElements());
4515 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
4516 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0),
4517 DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2);
4518 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
4519}
4520
4521SDValue
4522SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4523 SelectionDAG &DAG) const {
4524 // Handle extractions of floating-point values.
4525 SDLoc DL(Op);
4526 SDValue Op0 = Op.getOperand(0);
4527 SDValue Op1 = Op.getOperand(1);
4528 EVT VT = Op.getValueType();
4529 EVT VecVT = Op0.getValueType();
4530
4531 // Extractions of constant indices can be done directly.
4532 if (auto *CIndexN = dyn_cast<ConstantSDNode>(Op1)) {
4533 uint64_t Index = CIndexN->getZExtValue();
4534 unsigned Mask = VecVT.getVectorNumElements() - 1;
4535 if (Index <= Mask)
4536 return Op;
4537 }
4538
4539 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
4540 MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
4541 MVT IntVecVT = MVT::getVectorVT(IntVT, VecVT.getVectorNumElements());
4542 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT,
4543 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1);
4544 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
4545}
4546
4547SDValue
4548SystemZTargetLowering::lowerExtendVectorInreg(SDValue Op, SelectionDAG &DAG,
4549 unsigned UnpackHigh) const {
4550 SDValue PackedOp = Op.getOperand(0);
4551 EVT OutVT = Op.getValueType();
4552 EVT InVT = PackedOp.getValueType();
4553 unsigned ToBits = OutVT.getScalarSizeInBits();
4554 unsigned FromBits = InVT.getScalarSizeInBits();
4555 do {
4556 FromBits *= 2;
4557 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(FromBits),
4558 SystemZ::VectorBits / FromBits);
4559 PackedOp = DAG.getNode(UnpackHigh, SDLoc(PackedOp), OutVT, PackedOp);
4560 } while (FromBits != ToBits);
4561 return PackedOp;
4562}
4563
4564SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
4565 unsigned ByScalar) const {
4566 // Look for cases where a vector shift can use the *_BY_SCALAR form.
4567 SDValue Op0 = Op.getOperand(0);
4568 SDValue Op1 = Op.getOperand(1);
4569 SDLoc DL(Op);
4570 EVT VT = Op.getValueType();
4571 unsigned ElemBitSize = VT.getScalarSizeInBits();
4572
4573 // See whether the shift vector is a splat represented as BUILD_VECTOR.
4574 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op1)) {
4575 APInt SplatBits, SplatUndef;
4576 unsigned SplatBitSize;
4577 bool HasAnyUndefs;
4578 // Check for constant splats. Use ElemBitSize as the minimum element
4579 // width and reject splats that need wider elements.
4580 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
4581 ElemBitSize, true) &&
4582 SplatBitSize == ElemBitSize) {
4583 SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff,
4584 DL, MVT::i32);
4585 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
4586 }
4587 // Check for variable splats.
4588 BitVector UndefElements;
4589 SDValue Splat = BVN->getSplatValue(&UndefElements);
4590 if (Splat) {
4591 // Since i32 is the smallest legal type, we either need a no-op
4592 // or a truncation.
4593 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
4594 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
4595 }
4596 }
4597
4598 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
4599 // and the shift amount is directly available in a GPR.
4600 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Op1)) {
4601 if (VSN->isSplat()) {
4602 SDValue VSNOp0 = VSN->getOperand(0);
4603 unsigned Index = VSN->getSplatIndex();
4604 assert(Index < VT.getVectorNumElements() &&(static_cast <bool> (Index < VT.getVectorNumElements
() && "Splat index should be defined and in first operand"
) ? void (0) : __assert_fail ("Index < VT.getVectorNumElements() && \"Splat index should be defined and in first operand\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4605, __extension__ __PRETTY_FUNCTION__))
4605 "Splat index should be defined and in first operand")(static_cast <bool> (Index < VT.getVectorNumElements
() && "Splat index should be defined and in first operand"
) ? void (0) : __assert_fail ("Index < VT.getVectorNumElements() && \"Splat index should be defined and in first operand\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4605, __extension__ __PRETTY_FUNCTION__))
;
4606 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
4607 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
4608 // Since i32 is the smallest legal type, we either need a no-op
4609 // or a truncation.
4610 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
4611 VSNOp0.getOperand(Index));
4612 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
4613 }
4614 }
4615 }
4616
4617 // Otherwise just treat the current form as legal.
4618 return Op;
4619}
4620
4621SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
4622 SelectionDAG &DAG) const {
4623 switch (Op.getOpcode()) {
1
Control jumps to 'case BUILD_VECTOR:' at line 4706
4624 case ISD::FRAMEADDR:
4625 return lowerFRAMEADDR(Op, DAG);
4626 case ISD::RETURNADDR:
4627 return lowerRETURNADDR(Op, DAG);
4628 case ISD::BR_CC:
4629 return lowerBR_CC(Op, DAG);
4630 case ISD::SELECT_CC:
4631 return lowerSELECT_CC(Op, DAG);
4632 case ISD::SETCC:
4633 return lowerSETCC(Op, DAG);
4634 case ISD::GlobalAddress:
4635 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG);
4636 case ISD::GlobalTLSAddress:
4637 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG);
4638 case ISD::BlockAddress:
4639 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG);
4640 case ISD::JumpTable:
4641 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG);
4642 case ISD::ConstantPool:
4643 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG);
4644 case ISD::BITCAST:
4645 return lowerBITCAST(Op, DAG);
4646 case ISD::VASTART:
4647 return lowerVASTART(Op, DAG);
4648 case ISD::VACOPY:
4649 return lowerVACOPY(Op, DAG);
4650 case ISD::DYNAMIC_STACKALLOC:
4651 return lowerDYNAMIC_STACKALLOC(Op, DAG);
4652 case ISD::GET_DYNAMIC_AREA_OFFSET:
4653 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
4654 case ISD::SMUL_LOHI:
4655 return lowerSMUL_LOHI(Op, DAG);
4656 case ISD::UMUL_LOHI:
4657 return lowerUMUL_LOHI(Op, DAG);
4658 case ISD::SDIVREM:
4659 return lowerSDIVREM(Op, DAG);
4660 case ISD::UDIVREM:
4661 return lowerUDIVREM(Op, DAG);
4662 case ISD::OR:
4663 return lowerOR(Op, DAG);
4664 case ISD::CTPOP:
4665 return lowerCTPOP(Op, DAG);
4666 case ISD::ATOMIC_FENCE:
4667 return lowerATOMIC_FENCE(Op, DAG);
4668 case ISD::ATOMIC_SWAP:
4669 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
4670 case ISD::ATOMIC_STORE:
4671 return lowerATOMIC_STORE(Op, DAG);
4672 case ISD::ATOMIC_LOAD:
4673 return lowerATOMIC_LOAD(Op, DAG);
4674 case ISD::ATOMIC_LOAD_ADD:
4675 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
4676 case ISD::ATOMIC_LOAD_SUB:
4677 return lowerATOMIC_LOAD_SUB(Op, DAG);
4678 case ISD::ATOMIC_LOAD_AND:
4679 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND);
4680 case ISD::ATOMIC_LOAD_OR:
4681 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR);
4682 case ISD::ATOMIC_LOAD_XOR:
4683 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR);
4684 case ISD::ATOMIC_LOAD_NAND:
4685 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND);
4686 case ISD::ATOMIC_LOAD_MIN:
4687 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN);
4688 case ISD::ATOMIC_LOAD_MAX:
4689 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX);
4690 case ISD::ATOMIC_LOAD_UMIN:
4691 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
4692 case ISD::ATOMIC_LOAD_UMAX:
4693 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
4694 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4695 return lowerATOMIC_CMP_SWAP(Op, DAG);
4696 case ISD::STACKSAVE:
4697 return lowerSTACKSAVE(Op, DAG);
4698 case ISD::STACKRESTORE:
4699 return lowerSTACKRESTORE(Op, DAG);
4700 case ISD::PREFETCH:
4701 return lowerPREFETCH(Op, DAG);
4702 case ISD::INTRINSIC_W_CHAIN:
4703 return lowerINTRINSIC_W_CHAIN(Op, DAG);
4704 case ISD::INTRINSIC_WO_CHAIN:
4705 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
4706 case ISD::BUILD_VECTOR:
4707 return lowerBUILD_VECTOR(Op, DAG);
2
Calling 'SystemZTargetLowering::lowerBUILD_VECTOR'
4708 case ISD::VECTOR_SHUFFLE:
4709 return lowerVECTOR_SHUFFLE(Op, DAG);
4710 case ISD::SCALAR_TO_VECTOR:
4711 return lowerSCALAR_TO_VECTOR(Op, DAG);
4712 case ISD::INSERT_VECTOR_ELT:
4713 return lowerINSERT_VECTOR_ELT(Op, DAG);
4714 case ISD::EXTRACT_VECTOR_ELT:
4715 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
4716 case ISD::SIGN_EXTEND_VECTOR_INREG:
4717 return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACK_HIGH);
4718 case ISD::ZERO_EXTEND_VECTOR_INREG:
4719 return lowerExtendVectorInreg(Op, DAG, SystemZISD::UNPACKL_HIGH);
4720 case ISD::SHL:
4721 return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR);
4722 case ISD::SRL:
4723 return lowerShift(Op, DAG, SystemZISD::VSRL_BY_SCALAR);
4724 case ISD::SRA:
4725 return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR);
4726 default:
4727 llvm_unreachable("Unexpected node to lower")::llvm::llvm_unreachable_internal("Unexpected node to lower",
"/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4727)
;
4728 }
4729}
4730
4731// Lower operations with invalid operand or result types (currently used
4732// only for 128-bit integer types).
4733
4734static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
4735 SDLoc DL(In);
4736 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
4737 DAG.getIntPtrConstant(0, DL));
4738 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
4739 DAG.getIntPtrConstant(1, DL));
4740 SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL,
4741 MVT::Untyped, Hi, Lo);
4742 return SDValue(Pair, 0);
4743}
4744
4745static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
4746 SDLoc DL(In);
4747 SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
4748 DL, MVT::i64, In);
4749 SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
4750 DL, MVT::i64, In);
4751 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
4752}
4753
4754void
4755SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
4756 SmallVectorImpl<SDValue> &Results,
4757 SelectionDAG &DAG) const {
4758 switch (N->getOpcode()) {
4759 case ISD::ATOMIC_LOAD: {
4760 SDLoc DL(N);
4761 SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::Other);
4762 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
4763 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
4764 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_LOAD_128,
4765 DL, Tys, Ops, MVT::i128, MMO);
4766 Results.push_back(lowerGR128ToI128(DAG, Res));
4767 Results.push_back(Res.getValue(1));
4768 break;
4769 }
4770 case ISD::ATOMIC_STORE: {
4771 SDLoc DL(N);
4772 SDVTList Tys = DAG.getVTList(MVT::Other);
4773 SDValue Ops[] = { N->getOperand(0),
4774 lowerI128ToGR128(DAG, N->getOperand(2)),
4775 N->getOperand(1) };
4776 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
4777 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128,
4778 DL, Tys, Ops, MVT::i128, MMO);
4779 // We have to enforce sequential consistency by performing a
4780 // serialization operation after the store.
4781 if (cast<AtomicSDNode>(N)->getOrdering() ==
4782 AtomicOrdering::SequentiallyConsistent)
4783 Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL,
4784 MVT::Other, Res), 0);
4785 Results.push_back(Res);
4786 break;
4787 }
4788 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
4789 SDLoc DL(N);
4790 SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::Other, MVT::Glue);
4791 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
4792 lowerI128ToGR128(DAG, N->getOperand(2)),
4793 lowerI128ToGR128(DAG, N->getOperand(3)) };
4794 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
4795 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP_128,
4796 DL, Tys, Ops, MVT::i128, MMO);
4797 SDValue Success = emitSETCC(DAG, DL, Res.getValue(2),
4798 SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
4799 Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));
4800 Results.push_back(lowerGR128ToI128(DAG, Res));
4801 Results.push_back(Success);
4802 Results.push_back(Res.getValue(1));
4803 break;
4804 }
4805 default:
4806 llvm_unreachable("Unexpected node to lower")::llvm::llvm_unreachable_internal("Unexpected node to lower",
"/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 4806)
;
4807 }
4808}
4809
4810void
4811SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
4812 SmallVectorImpl<SDValue> &Results,
4813 SelectionDAG &DAG) const {
4814 return LowerOperationWrapper(N, Results, DAG);
4815}
4816
4817const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
4818#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME
4819 switch ((SystemZISD::NodeType)Opcode) {
4820 case SystemZISD::FIRST_NUMBER: break;
4821 OPCODE(RET_FLAG);
4822 OPCODE(CALL);
4823 OPCODE(SIBCALL);
4824 OPCODE(TLS_GDCALL);
4825 OPCODE(TLS_LDCALL);
4826 OPCODE(PCREL_WRAPPER);
4827 OPCODE(PCREL_OFFSET);
4828 OPCODE(IABS);
4829 OPCODE(ICMP);
4830 OPCODE(FCMP);
4831 OPCODE(TM);
4832 OPCODE(BR_CCMASK);
4833 OPCODE(SELECT_CCMASK);
4834 OPCODE(ADJDYNALLOC);
4835 OPCODE(POPCNT);
4836 OPCODE(SMUL_LOHI);
4837 OPCODE(UMUL_LOHI);
4838 OPCODE(SDIVREM);
4839 OPCODE(UDIVREM);
4840 OPCODE(MVC);
4841 OPCODE(MVC_LOOP);
4842 OPCODE(NC);
4843 OPCODE(NC_LOOP);
4844 OPCODE(OC);
4845 OPCODE(OC_LOOP);
4846 OPCODE(XC);
4847 OPCODE(XC_LOOP);
4848 OPCODE(CLC);
4849 OPCODE(CLC_LOOP);
4850 OPCODE(STPCPY);
4851 OPCODE(STRCMP);
4852 OPCODE(SEARCH_STRING);
4853 OPCODE(IPM);
4854 OPCODE(MEMBARRIER);
4855 OPCODE(TBEGIN);
4856 OPCODE(TBEGIN_NOFLOAT);
4857 OPCODE(TEND);
4858 OPCODE(BYTE_MASK);
4859 OPCODE(ROTATE_MASK);
4860 OPCODE(REPLICATE);
4861 OPCODE(JOIN_DWORDS);
4862 OPCODE(SPLAT);
4863 OPCODE(MERGE_HIGH);
4864 OPCODE(MERGE_LOW);
4865 OPCODE(SHL_DOUBLE);
4866 OPCODE(PERMUTE_DWORDS);
4867 OPCODE(PERMUTE);
4868 OPCODE(PACK);
4869 OPCODE(PACKS_CC);
4870 OPCODE(PACKLS_CC);
4871 OPCODE(UNPACK_HIGH);
4872 OPCODE(UNPACKL_HIGH);
4873 OPCODE(UNPACK_LOW);
4874 OPCODE(UNPACKL_LOW);
4875 OPCODE(VSHL_BY_SCALAR);
4876 OPCODE(VSRL_BY_SCALAR);
4877 OPCODE(VSRA_BY_SCALAR);
4878 OPCODE(VSUM);
4879 OPCODE(VICMPE);
4880 OPCODE(VICMPH);
4881 OPCODE(VICMPHL);
4882 OPCODE(VICMPES);
4883 OPCODE(VICMPHS);
4884 OPCODE(VICMPHLS);
4885 OPCODE(VFCMPE);
4886 OPCODE(VFCMPH);
4887 OPCODE(VFCMPHE);
4888 OPCODE(VFCMPES);
4889 OPCODE(VFCMPHS);
4890 OPCODE(VFCMPHES);
4891 OPCODE(VFTCI);
4892 OPCODE(VEXTEND);
4893 OPCODE(VROUND);
4894 OPCODE(VTM);
4895 OPCODE(VFAE_CC);
4896 OPCODE(VFAEZ_CC);
4897 OPCODE(VFEE_CC);
4898 OPCODE(VFEEZ_CC);
4899 OPCODE(VFENE_CC);
4900 OPCODE(VFENEZ_CC);
4901 OPCODE(VISTR_CC);
4902 OPCODE(VSTRC_CC);
4903 OPCODE(VSTRCZ_CC);
4904 OPCODE(TDC);
4905 OPCODE(ATOMIC_SWAPW);
4906 OPCODE(ATOMIC_LOADW_ADD);
4907 OPCODE(ATOMIC_LOADW_SUB);
4908 OPCODE(ATOMIC_LOADW_AND);
4909 OPCODE(ATOMIC_LOADW_OR);
4910 OPCODE(ATOMIC_LOADW_XOR);
4911 OPCODE(ATOMIC_LOADW_NAND);
4912 OPCODE(ATOMIC_LOADW_MIN);
4913 OPCODE(ATOMIC_LOADW_MAX);
4914 OPCODE(ATOMIC_LOADW_UMIN);
4915 OPCODE(ATOMIC_LOADW_UMAX);
4916 OPCODE(ATOMIC_CMP_SWAPW);
4917 OPCODE(ATOMIC_CMP_SWAP);
4918 OPCODE(ATOMIC_LOAD_128);
4919 OPCODE(ATOMIC_STORE_128);
4920 OPCODE(ATOMIC_CMP_SWAP_128);
4921 OPCODE(LRV);
4922 OPCODE(STRV);
4923 OPCODE(PREFETCH);
4924 }
4925 return nullptr;
4926#undef OPCODE
4927}
4928
4929// Return true if VT is a vector whose elements are a whole number of bytes
4930// in width. Also check for presence of vector support.
4931bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
4932 if (!Subtarget.hasVector())
4933 return false;
4934
4935 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
4936}
4937
4938// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
4939// producing a result of type ResVT. Op is a possibly bitcast version
4940// of the input vector and Index is the index (based on type VecVT) that
4941// should be extracted. Return the new extraction if a simplification
4942// was possible or if Force is true.
4943SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
4944 EVT VecVT, SDValue Op,
4945 unsigned Index,
4946 DAGCombinerInfo &DCI,
4947 bool Force) const {
4948 SelectionDAG &DAG = DCI.DAG;
4949
4950 // The number of bytes being extracted.
4951 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
4952
4953 for (;;) {
4954 unsigned Opcode = Op.getOpcode();
4955 if (Opcode == ISD::BITCAST)
4956 // Look through bitcasts.
4957 Op = Op.getOperand(0);
4958 else if (Opcode == ISD::VECTOR_SHUFFLE &&
4959 canTreatAsByteVector(Op.getValueType())) {
4960 // Get a VPERM-like permute mask and see whether the bytes covered
4961 // by the extracted element are a contiguous sequence from one
4962 // source operand.
4963 SmallVector<int, SystemZ::VectorBytes> Bytes;
4964 getVPermMask(cast<ShuffleVectorSDNode>(Op), Bytes);
4965 int First;
4966 if (!getShuffleInput(Bytes, Index * BytesPerElement,
4967 BytesPerElement, First))
4968 break;
4969 if (First < 0)
4970 return DAG.getUNDEF(ResVT);
4971 // Make sure the contiguous sequence starts at a multiple of the
4972 // original element size.
4973 unsigned Byte = unsigned(First) % Bytes.size();
4974 if (Byte % BytesPerElement != 0)
4975 break;
4976 // We can get the extracted value directly from an input.
4977 Index = Byte / BytesPerElement;
4978 Op = Op.getOperand(unsigned(First) / Bytes.size());
4979 Force = true;
4980 } else if (Opcode == ISD::BUILD_VECTOR &&
4981 canTreatAsByteVector(Op.getValueType())) {
4982 // We can only optimize this case if the BUILD_VECTOR elements are
4983 // at least as wide as the extracted value.
4984 EVT OpVT = Op.getValueType();
4985 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
4986 if (OpBytesPerElement < BytesPerElement)
4987 break;
4988 // Make sure that the least-significant bit of the extracted value
4989 // is the least significant bit of an input.
4990 unsigned End = (Index + 1) * BytesPerElement;
4991 if (End % OpBytesPerElement != 0)
4992 break;
4993 // We're extracting the low part of one operand of the BUILD_VECTOR.
4994 Op = Op.getOperand(End / OpBytesPerElement - 1);
4995 if (!Op.getValueType().isInteger()) {
4996 EVT VT = MVT::getIntegerVT(Op.getValueSizeInBits());
4997 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
4998 DCI.AddToWorklist(Op.getNode());
4999 }
5000 EVT VT = MVT::getIntegerVT(ResVT.getSizeInBits());
5001 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
5002 if (VT != ResVT) {
5003 DCI.AddToWorklist(Op.getNode());
5004 Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op);
5005 }
5006 return Op;
5007 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
5008 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
5009 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
5010 canTreatAsByteVector(Op.getValueType()) &&
5011 canTreatAsByteVector(Op.getOperand(0).getValueType())) {
5012 // Make sure that only the unextended bits are significant.
5013 EVT ExtVT = Op.getValueType();
5014 EVT OpVT = Op.getOperand(0).getValueType();
5015 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
5016 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
5017 unsigned Byte = Index * BytesPerElement;
5018 unsigned SubByte = Byte % ExtBytesPerElement;
5019 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
5020 if (SubByte < MinSubByte ||
5021 SubByte + BytesPerElement > ExtBytesPerElement)
5022 break;
5023 // Get the byte offset of the unextended element
5024 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
5025 // ...then add the byte offset relative to that element.
5026 Byte += SubByte - MinSubByte;
5027 if (Byte % BytesPerElement != 0)
5028 break;
5029 Op = Op.getOperand(0);
5030 Index = Byte / BytesPerElement;
5031 Force = true;
5032 } else
5033 break;
5034 }
5035 if (Force) {
5036 if (Op.getValueType() != VecVT) {
5037 Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op);
5038 DCI.AddToWorklist(Op.getNode());
5039 }
5040 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op,
5041 DAG.getConstant(Index, DL, MVT::i32));
5042 }
5043 return SDValue();
5044}
5045
5046// Optimize vector operations in scalar value Op on the basis that Op
5047// is truncated to TruncVT.
5048SDValue SystemZTargetLowering::combineTruncateExtract(
5049 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
5050 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
5051 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
5052 // of type TruncVT.
5053 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5054 TruncVT.getSizeInBits() % 8 == 0) {
5055 SDValue Vec = Op.getOperand(0);
5056 EVT VecVT = Vec.getValueType();
5057 if (canTreatAsByteVector(VecVT)) {
5058 if (auto *IndexN = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
5059 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
5060 unsigned TruncBytes = TruncVT.getStoreSize();
5061 if (BytesPerElement % TruncBytes == 0) {
5062 // Calculate the value of Y' in the above description. We are
5063 // splitting the original elements into Scale equal-sized pieces
5064 // and for truncation purposes want the last (least-significant)
5065 // of these pieces for IndexN. This is easiest to do by calculating
5066 // the start index of the following element and then subtracting 1.
5067 unsigned Scale = BytesPerElement / TruncBytes;
5068 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
5069
5070 // Defer the creation of the bitcast from X to combineExtract,
5071 // which might be able to optimize the extraction.
5072 VecVT = MVT::getVectorVT(MVT::getIntegerVT(TruncBytes * 8),
5073 VecVT.getStoreSize() / TruncBytes);
5074 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
5075 return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true);
5076 }
5077 }
5078 }
5079 }
5080 return SDValue();
5081}
5082
5083SDValue SystemZTargetLowering::combineZERO_EXTEND(
5084 SDNode *N, DAGCombinerInfo &DCI) const {
5085 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
5086 SelectionDAG &DAG = DCI.DAG;
5087 SDValue N0 = N->getOperand(0);
5088 EVT VT = N->getValueType(0);
5089 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
5090 auto *TrueOp = dyn_cast<ConstantSDNode>(N0.getOperand(0));
5091 auto *FalseOp = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5092 if (TrueOp && FalseOp) {
5093 SDLoc DL(N0);
5094 SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),
5095 DAG.getConstant(FalseOp->getZExtValue(), DL, VT),
5096 N0.getOperand(2), N0.getOperand(3), N0.getOperand(4) };
5097 SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops);
5098 // If N0 has multiple uses, change other uses as well.
5099 if (!N0.hasOneUse()) {
5100 SDValue TruncSelect =
5101 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), NewSelect);
5102 DCI.CombineTo(N0.getNode(), TruncSelect);
5103 }
5104 return NewSelect;
5105 }
5106 }
5107 return SDValue();
5108}
5109
5110SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
5111 SDNode *N, DAGCombinerInfo &DCI) const {
5112 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
5113 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
5114 // into (select_cc LHS, RHS, -1, 0, COND)
5115 SelectionDAG &DAG = DCI.DAG;
5116 SDValue N0 = N->getOperand(0);
5117 EVT VT = N->getValueType(0);
5118 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
5119 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
5120 N0 = N0.getOperand(0);
5121 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
5122 SDLoc DL(N0);
5123 SDValue Ops[] = { N0.getOperand(0), N0.getOperand(1),
5124 DAG.getConstant(-1, DL, VT), DAG.getConstant(0, DL, VT),
5125 N0.getOperand(2) };
5126 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
5127 }
5128 return SDValue();
5129}
5130
5131SDValue SystemZTargetLowering::combineSIGN_EXTEND(
5132 SDNode *N, DAGCombinerInfo &DCI) const {
5133 // Convert (sext (ashr (shl X, C1), C2)) to
5134 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
5135 // cheap as narrower ones.
5136 SelectionDAG &DAG = DCI.DAG;
5137 SDValue N0 = N->getOperand(0);
5138 EVT VT = N->getValueType(0);
5139 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
5140 auto *SraAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5141 SDValue Inner = N0.getOperand(0);
5142 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
5143 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.getOperand(1))) {
5144 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
5145 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
5146 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
5147 EVT ShiftVT = N0.getOperand(1).getValueType();
5148 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT,
5149 Inner.getOperand(0));
5150 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext,
5151 DAG.getConstant(NewShlAmt, SDLoc(Inner),
5152 ShiftVT));
5153 return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl,
5154 DAG.getConstant(NewSraAmt, SDLoc(N0), ShiftVT));
5155 }
5156 }
5157 }
5158 return SDValue();
5159}
5160
5161SDValue SystemZTargetLowering::combineMERGE(
5162 SDNode *N, DAGCombinerInfo &DCI) const {
5163 SelectionDAG &DAG = DCI.DAG;
5164 unsigned Opcode = N->getOpcode();
5165 SDValue Op0 = N->getOperand(0);
5166 SDValue Op1 = N->getOperand(1);
5167 if (Op0.getOpcode() == ISD::BITCAST)
5168 Op0 = Op0.getOperand(0);
5169 if (Op0.getOpcode() == SystemZISD::BYTE_MASK &&
5170 cast<ConstantSDNode>(Op0.getOperand(0))->getZExtValue() == 0) {
5171 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
5172 // for v4f32.
5173 if (Op1 == N->getOperand(0))
5174 return Op1;
5175 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
5176 EVT VT = Op1.getValueType();
5177 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
5178 if (ElemBytes <= 4) {
5179 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
5180 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
5181 EVT InVT = VT.changeVectorElementTypeToInteger();
5182 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(ElemBytes * 16),
5183 SystemZ::VectorBytes / ElemBytes / 2);
5184 if (VT != InVT) {
5185 Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), InVT, Op1);
5186 DCI.AddToWorklist(Op1.getNode());
5187 }
5188 SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1);
5189 DCI.AddToWorklist(Op.getNode());
5190 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
5191 }
5192 }
5193 return SDValue();
5194}
5195
5196SDValue SystemZTargetLowering::combineSTORE(
5197 SDNode *N, DAGCombinerInfo &DCI) const {
5198 SelectionDAG &DAG = DCI.DAG;
5199 auto *SN = cast<StoreSDNode>(N);
5200 auto &Op1 = N->getOperand(1);
5201 EVT MemVT = SN->getMemoryVT();
5202 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
5203 // for the extraction to be done on a vMiN value, so that we can use VSTE.
5204 // If X has wider elements then convert it to:
5205 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
5206 if (MemVT.isInteger()) {
5207 if (SDValue Value =
5208 combineTruncateExtract(SDLoc(N), MemVT, SN->getValue(), DCI)) {
5209 DCI.AddToWorklist(Value.getNode());
5210
5211 // Rewrite the store with the new form of stored value.
5212 return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value,
5213 SN->getBasePtr(), SN->getMemoryVT(),
5214 SN->getMemOperand());
5215 }
5216 }
5217 // Combine STORE (BSWAP) into STRVH/STRV/STRVG
5218 // See comment in combineBSWAP about volatile accesses.
5219 if (!SN->isTruncatingStore() &&
5220 !SN->isVolatile() &&
5221 Op1.getOpcode() == ISD::BSWAP &&
5222 Op1.getNode()->hasOneUse() &&
5223 (Op1.getValueType() == MVT::i16 ||
5224 Op1.getValueType() == MVT::i32 ||
5225 Op1.getValueType() == MVT::i64)) {
5226
5227 SDValue BSwapOp = Op1.getOperand(0);
5228
5229 if (BSwapOp.getValueType() == MVT::i16)
5230 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), MVT::i32, BSwapOp);
5231
5232 SDValue Ops[] = {
5233 N->getOperand(0), BSwapOp, N->getOperand(2),
5234 DAG.getValueType(Op1.getValueType())
5235 };
5236
5237 return
5238 DAG.getMemIntrinsicNode(SystemZISD::STRV, SDLoc(N), DAG.getVTList(MVT::Other),
5239 Ops, MemVT, SN->getMemOperand());
5240 }
5241 return SDValue();
5242}
5243
5244SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
5245 SDNode *N, DAGCombinerInfo &DCI) const {
5246
5247 if (!Subtarget.hasVector())
5248 return SDValue();
5249
5250 // Try to simplify a vector extraction.
5251 if (auto *IndexN = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
5252 SDValue Op0 = N->getOperand(0);
5253 EVT VecVT = Op0.getValueType();
5254 return combineExtract(SDLoc(N), N->getValueType(0), VecVT, Op0,
5255 IndexN->getZExtValue(), DCI, false);
5256 }
5257 return SDValue();
5258}
5259
5260SDValue SystemZTargetLowering::combineJOIN_DWORDS(
5261 SDNode *N, DAGCombinerInfo &DCI) const {
5262 SelectionDAG &DAG = DCI.DAG;
5263 // (join_dwords X, X) == (replicate X)
5264 if (N->getOperand(0) == N->getOperand(1))
5265 return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0),
5266 N->getOperand(0));
5267 return SDValue();
5268}
5269
5270SDValue SystemZTargetLowering::combineFP_ROUND(
5271 SDNode *N, DAGCombinerInfo &DCI) const {
5272 // (fpround (extract_vector_elt X 0))
5273 // (fpround (extract_vector_elt X 1)) ->
5274 // (extract_vector_elt (VROUND X) 0)
5275 // (extract_vector_elt (VROUND X) 1)
5276 //
5277 // This is a special case since the target doesn't really support v2f32s.
5278 SelectionDAG &DAG = DCI.DAG;
5279 SDValue Op0 = N->getOperand(0);
5280 if (N->getValueType(0) == MVT::f32 &&
5281 Op0.hasOneUse() &&
5282 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5283 Op0.getOperand(0).getValueType() == MVT::v2f64 &&
5284 Op0.getOperand(1).getOpcode() == ISD::Constant &&
5285 cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue() == 0) {
5286 SDValue Vec = Op0.getOperand(0);
5287 for (auto *U : Vec->uses()) {
5288 if (U != Op0.getNode() &&
5289 U->hasOneUse() &&
5290 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5291 U->getOperand(0) == Vec &&
5292 U->getOperand(1).getOpcode() == ISD::Constant &&
5293 cast<ConstantSDNode>(U->getOperand(1))->getZExtValue() == 1) {
5294 SDValue OtherRound = SDValue(*U->use_begin(), 0);
5295 if (OtherRound.getOpcode() == ISD::FP_ROUND &&
5296 OtherRound.getOperand(0) == SDValue(U, 0) &&
5297 OtherRound.getValueType() == MVT::f32) {
5298 SDValue VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N),
5299 MVT::v4f32, Vec);
5300 DCI.AddToWorklist(VRound.getNode());
5301 SDValue Extract1 =
5302 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f32,
5303 VRound, DAG.getConstant(2, SDLoc(U), MVT::i32));
5304 DCI.AddToWorklist(Extract1.getNode());
5305 DAG.ReplaceAllUsesOfValueWith(OtherRound, Extract1);
5306 SDValue Extract0 =
5307 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f32,
5308 VRound, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
5309 return Extract0;
5310 }
5311 }
5312 }
5313 }
5314 return SDValue();
5315}
5316
5317SDValue SystemZTargetLowering::combineBSWAP(
5318 SDNode *N, DAGCombinerInfo &DCI) const {
5319 SelectionDAG &DAG = DCI.DAG;
5320 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG
5321 // These loads are allowed to access memory multiple times, and so we must check
5322 // that the loads are not volatile before performing the combine.
5323 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
5324 N->getOperand(0).hasOneUse() &&
5325 (N->getValueType(0) == MVT::i16 || N->getValueType(0) == MVT::i32 ||
5326 N->getValueType(0) == MVT::i64) &&
5327 !cast<LoadSDNode>(N->getOperand(0))->isVolatile()) {
5328 SDValue Load = N->getOperand(0);
5329 LoadSDNode *LD = cast<LoadSDNode>(Load);
5330
5331 // Create the byte-swapping load.
5332 SDValue Ops[] = {
5333 LD->getChain(), // Chain
5334 LD->getBasePtr(), // Ptr
5335 DAG.getValueType(N->getValueType(0)) // VT
5336 };
5337 SDValue BSLoad =
5338 DAG.getMemIntrinsicNode(SystemZISD::LRV, SDLoc(N),
5339 DAG.getVTList(N->getValueType(0) == MVT::i64 ?
5340 MVT::i64 : MVT::i32, MVT::Other),
5341 Ops, LD->getMemoryVT(), LD->getMemOperand());
5342
5343 // If this is an i16 load, insert the truncate.
5344 SDValue ResVal = BSLoad;
5345 if (N->getValueType(0) == MVT::i16)
5346 ResVal = DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16, BSLoad);
5347
5348 // First, combine the bswap away. This makes the value produced by the
5349 // load dead.
5350 DCI.CombineTo(N, ResVal);
5351
5352 // Next, combine the load away, we give it a bogus result value but a real
5353 // chain result. The result value is dead because the bswap is dead.
5354 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1));
5355
5356 // Return N so it doesn't get rechecked!
5357 return SDValue(N, 0);
5358 }
5359 return SDValue();
5360}
5361
5362SDValue SystemZTargetLowering::combineSHIFTROT(
5363 SDNode *N, DAGCombinerInfo &DCI) const {
5364
5365 SelectionDAG &DAG = DCI.DAG;
5366
5367 // Shift/rotate instructions only use the last 6 bits of the second operand
5368 // register. If the second operand is the result of an AND with an immediate
5369 // value that has its last 6 bits set, we can safely remove the AND operation.
5370 //
5371 // If the AND operation doesn't have the last 6 bits set, we can't remove it
5372 // entirely, but we can still truncate it to a 16-bit value. This prevents
5373 // us from ending up with a NILL with a signed operand, which will cause the
5374 // instruction printer to abort.
5375 SDValue N1 = N->getOperand(1);
5376 if (N1.getOpcode() == ISD::AND) {
5377 SDValue AndMaskOp = N1->getOperand(1);
5378 auto *AndMask = dyn_cast<ConstantSDNode>(AndMaskOp);
5379
5380 // The AND mask is constant
5381 if (AndMask) {
5382 auto AmtVal = AndMask->getZExtValue();
5383
5384 // Bottom 6 bits are set
5385 if ((AmtVal & 0x3f) == 0x3f) {
5386 SDValue AndOp = N1->getOperand(0);
5387
5388 // This is the only use, so remove the node
5389 if (N1.hasOneUse()) {
5390 // Combine the AND away
5391 DCI.CombineTo(N1.getNode(), AndOp);
5392
5393 // Return N so it isn't rechecked
5394 return SDValue(N, 0);
5395
5396 // The node will be reused, so create a new node for this one use
5397 } else {
5398 SDValue Replace = DAG.getNode(N->getOpcode(), SDLoc(N),
5399 N->getValueType(0), N->getOperand(0),
5400 AndOp);
5401 DCI.AddToWorklist(Replace.getNode());
5402
5403 return Replace;
5404 }
5405
5406 // We can't remove the AND, but we can use NILL here (normally we would
5407 // use NILF). Only keep the last 16 bits of the mask. The actual
5408 // transformation will be handled by .td definitions.
5409 } else if (AmtVal >> 16 != 0) {
5410 SDValue AndOp = N1->getOperand(0);
5411
5412 auto NewMask = DAG.getConstant(AndMask->getZExtValue() & 0x0000ffff,
5413 SDLoc(AndMaskOp),
5414 AndMaskOp.getValueType());
5415
5416 auto NewAnd = DAG.getNode(N1.getOpcode(), SDLoc(N1), N1.getValueType(),
5417 AndOp, NewMask);
5418
5419 SDValue Replace = DAG.getNode(N->getOpcode(), SDLoc(N),
5420 N->getValueType(0), N->getOperand(0),
5421 NewAnd);
5422 DCI.AddToWorklist(Replace.getNode());
5423
5424 return Replace;
5425 }
5426 }
5427 }
5428
5429 return SDValue();
5430}
5431
5432static bool combineCCMask(SDValue &Glue, int &CCValid, int &CCMask) {
5433 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
5434 // set by the glued instruction using the CCValid / CCMask masks,
5435 // If the glued instruction is itself a (ICMP (SELECT_CCMASK)) testing
5436 // the condition code set by some other instruction, see whether we
5437 // can directly use that condition code.
5438 bool Invert = false;
5439
5440 // Verify that we have an appropriate mask for a EQ or NE comparison.
5441 if (CCValid != SystemZ::CCMASK_ICMP)
5442 return false;
5443 if (CCMask == SystemZ::CCMASK_CMP_NE)
5444 Invert = !Invert;
5445 else if (CCMask != SystemZ::CCMASK_CMP_EQ)
5446 return false;
5447
5448 // Verify that we have an ICMP that is the single user of a SELECT_CCMASK.
5449 SDNode *ICmp = Glue.getNode();
5450 if (ICmp->getOpcode() != SystemZISD::ICMP)
5451 return false;
5452 SDNode *Select = ICmp->getOperand(0).getNode();
5453 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
5454 return false;
5455 if (!Select->hasOneUse())
5456 return false;
5457
5458 // Verify that the ICMP compares against one of select values.
5459 auto *CompareVal = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
5460 if (!CompareVal)
5461 return false;
5462 auto *TrueVal = dyn_cast<ConstantSDNode>(Select->getOperand(0));
5463 if (!TrueVal)
5464 return false;
5465 auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1));
5466 if (!FalseVal)
5467 return false;
5468 if (CompareVal->getZExtValue() == FalseVal->getZExtValue())
5469 Invert = !Invert;
5470 else if (CompareVal->getZExtValue() != TrueVal->getZExtValue())
5471 return false;
5472
5473 // Compute the effective CC mask for the new branch or select.
5474 auto *NewCCValid = dyn_cast<ConstantSDNode>(Select->getOperand(2));
5475 auto *NewCCMask = dyn_cast<ConstantSDNode>(Select->getOperand(3));
5476 if (!NewCCValid || !NewCCMask)
5477 return false;
5478 CCValid = NewCCValid->getZExtValue();
5479 CCMask = NewCCMask->getZExtValue();
5480 if (Invert)
5481 CCMask ^= CCValid;
5482
5483 // Return the updated Glue link.
5484 Glue = Select->getOperand(4);
5485 return true;
5486}
5487
5488static bool combineMergeChains(SDValue &Chain, SDValue Glue) {
5489 // We are about to glue an instruction with input chain Chain to the
5490 // instruction Glue. Verify that this would not create an invalid
5491 // topological sort due to intervening chain nodes.
5492
5493 SDNode *Node = Glue.getNode();
5494 for (int ResNo = Node->getNumValues() - 1; ResNo >= 0; --ResNo)
5495 if (Node->getValueType(ResNo) == MVT::Other) {
5496 SDValue OutChain = SDValue(Node, ResNo);
5497 // FIXME: We should be able to at least handle an intervening
5498 // TokenFactor node by swapping chains around a bit ...
5499 return Chain == OutChain;
5500 }
5501
5502 return true;
5503}
5504
5505SDValue SystemZTargetLowering::combineBR_CCMASK(
5506 SDNode *N, DAGCombinerInfo &DCI) const {
5507 SelectionDAG &DAG = DCI.DAG;
5508
5509 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
5510 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));
5511 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));
5512 if (!CCValid || !CCMask)
5513 return SDValue();
5514
5515 int CCValidVal = CCValid->getZExtValue();
5516 int CCMaskVal = CCMask->getZExtValue();
5517 SDValue Chain = N->getOperand(0);
5518 SDValue Glue = N->getOperand(4);
5519
5520 if (combineCCMask(Glue, CCValidVal, CCMaskVal)
5521 && combineMergeChains(Chain, Glue))
5522 return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0),
5523 Chain,
5524 DAG.getConstant(CCValidVal, SDLoc(N), MVT::i32),
5525 DAG.getConstant(CCMaskVal, SDLoc(N), MVT::i32),
5526 N->getOperand(3), Glue);
5527 return SDValue();
5528}
5529
5530SDValue SystemZTargetLowering::combineSELECT_CCMASK(
5531 SDNode *N, DAGCombinerInfo &DCI) const {
5532 SelectionDAG &DAG = DCI.DAG;
5533
5534 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
5535 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
5536 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
5537 if (!CCValid || !CCMask)
5538 return SDValue();
5539
5540 int CCValidVal = CCValid->getZExtValue();
5541 int CCMaskVal = CCMask->getZExtValue();
5542 SDValue Glue = N->getOperand(4);
5543
5544 if (combineCCMask(Glue, CCValidVal, CCMaskVal))
5545 return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
5546 N->getOperand(0),
5547 N->getOperand(1),
5548 DAG.getConstant(CCValidVal, SDLoc(N), MVT::i32),
5549 DAG.getConstant(CCMaskVal, SDLoc(N), MVT::i32),
5550 Glue);
5551 return SDValue();
5552}
5553
5554SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
5555 DAGCombinerInfo &DCI) const {
5556 switch(N->getOpcode()) {
5557 default: break;
5558 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
5559 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
5560 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
5561 case SystemZISD::MERGE_HIGH:
5562 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
5563 case ISD::STORE: return combineSTORE(N, DCI);
5564 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
5565 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
5566 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
5567 case ISD::BSWAP: return combineBSWAP(N, DCI);
5568 case ISD::SHL:
5569 case ISD::SRA:
5570 case ISD::SRL:
5571 case ISD::ROTL: return combineSHIFTROT(N, DCI);
5572 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
5573 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
5574 }
5575
5576 return SDValue();
5577}
5578
5579void
5580SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
5581 KnownBits &Known,
5582 const APInt &DemandedElts,
5583 const SelectionDAG &DAG,
5584 unsigned Depth) const {
5585 unsigned BitWidth = Known.getBitWidth();
5586
5587 Known.resetAll();
5588 switch (Op.getOpcode()) {
5589 case SystemZISD::SELECT_CCMASK: {
5590 KnownBits TrueKnown(BitWidth), FalseKnown(BitWidth);
5591 DAG.computeKnownBits(Op.getOperand(0), TrueKnown, Depth + 1);
5592 DAG.computeKnownBits(Op.getOperand(1), FalseKnown, Depth + 1);
5593 Known.Zero = TrueKnown.Zero & FalseKnown.Zero;
5594 Known.One = TrueKnown.One & FalseKnown.One;
5595 break;
5596 }
5597
5598 default:
5599 break;
5600 }
5601}
5602
5603//===----------------------------------------------------------------------===//
5604// Custom insertion
5605//===----------------------------------------------------------------------===//
5606
5607// Create a new basic block after MBB.
5608static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) {
5609 MachineFunction &MF = *MBB->getParent();
5610 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock());
5611 MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB);
5612 return NewMBB;
5613}
5614
5615// Split MBB after MI and return the new block (the one that contains
5616// instructions after MI).
5617static MachineBasicBlock *splitBlockAfter(MachineBasicBlock::iterator MI,
5618 MachineBasicBlock *MBB) {
5619 MachineBasicBlock *NewMBB = emitBlockAfter(MBB);
5620 NewMBB->splice(NewMBB->begin(), MBB,
5621 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
5622 NewMBB->transferSuccessorsAndUpdatePHIs(MBB);
5623 return NewMBB;
5624}
5625
5626// Split MBB before MI and return the new block (the one that contains MI).
5627static MachineBasicBlock *splitBlockBefore(MachineBasicBlock::iterator MI,
5628 MachineBasicBlock *MBB) {
5629 MachineBasicBlock *NewMBB = emitBlockAfter(MBB);
5630 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end());
5631 NewMBB->transferSuccessorsAndUpdatePHIs(MBB);
5632 return NewMBB;
5633}
5634
5635// Force base value Base into a register before MI. Return the register.
5636static unsigned forceReg(MachineInstr &MI, MachineOperand &Base,
5637 const SystemZInstrInfo *TII) {
5638 if (Base.isReg())
5639 return Base.getReg();
5640
5641 MachineBasicBlock *MBB = MI.getParent();
5642 MachineFunction &MF = *MBB->getParent();
5643 MachineRegisterInfo &MRI = MF.getRegInfo();
5644
5645 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
5646 BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LA), Reg)
5647 .add(Base)
5648 .addImm(0)
5649 .addReg(0);
5650 return Reg;
5651}
5652
5653// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
5654MachineBasicBlock *
5655SystemZTargetLowering::emitSelect(MachineInstr &MI,
5656 MachineBasicBlock *MBB,
5657 unsigned LOCROpcode) const {
5658 const SystemZInstrInfo *TII =
5659 static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
5660
5661 unsigned DestReg = MI.getOperand(0).getReg();
5662 unsigned TrueReg = MI.getOperand(1).getReg();
5663 unsigned FalseReg = MI.getOperand(2).getReg();
5664 unsigned CCValid = MI.getOperand(3).getImm();
5665 unsigned CCMask = MI.getOperand(4).getImm();
5666 DebugLoc DL = MI.getDebugLoc();
5667
5668 // Use LOCROpcode if possible.
5669 if (LOCROpcode && Subtarget.hasLoadStoreOnCond()) {
5670 BuildMI(*MBB, MI, DL, TII->get(LOCROpcode), DestReg)
5671 .addReg(FalseReg).addReg(TrueReg)
5672 .addImm(CCValid).addImm(CCMask);
5673 MI.eraseFromParent();
5674 return MBB;
5675 }
5676
5677 MachineBasicBlock *StartMBB = MBB;
5678 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB);
5679 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB);
5680
5681 // StartMBB:
5682 // BRC CCMask, JoinMBB
5683 // # fallthrough to FalseMBB
5684 MBB = StartMBB;
5685 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5686 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
5687 MBB->addSuccessor(JoinMBB);
5688 MBB->addSuccessor(FalseMBB);
5689
5690 // FalseMBB:
5691 // # fallthrough to JoinMBB
5692 MBB = FalseMBB;
5693 MBB->addSuccessor(JoinMBB);
5694
5695 // JoinMBB:
5696 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
5697 // ...
5698 MBB = JoinMBB;
5699 BuildMI(*MBB, MI, DL, TII->get(SystemZ::PHI), DestReg)
5700 .addReg(TrueReg).addMBB(StartMBB)
5701 .addReg(FalseReg).addMBB(FalseMBB);
5702
5703 MI.eraseFromParent();
5704 return JoinMBB;
5705}
5706
5707// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
5708// StoreOpcode is the store to use and Invert says whether the store should
5709// happen when the condition is false rather than true. If a STORE ON
5710// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
5711MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
5712 MachineBasicBlock *MBB,
5713 unsigned StoreOpcode,
5714 unsigned STOCOpcode,
5715 bool Invert) const {
5716 const SystemZInstrInfo *TII =
5717 static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
5718
5719 unsigned SrcReg = MI.getOperand(0).getReg();
5720 MachineOperand Base = MI.getOperand(1);
5721 int64_t Disp = MI.getOperand(2).getImm();
5722 unsigned IndexReg = MI.getOperand(3).getReg();
5723 unsigned CCValid = MI.getOperand(4).getImm();
5724 unsigned CCMask = MI.getOperand(5).getImm();
5725 DebugLoc DL = MI.getDebugLoc();
5726
5727 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp);
5728
5729 // Use STOCOpcode if possible. We could use different store patterns in
5730 // order to avoid matching the index register, but the performance trade-offs
5731 // might be more complicated in that case.
5732 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
5733 if (Invert)
5734 CCMask ^= CCValid;
5735
5736 // ISel pattern matching also adds a load memory operand of the same
5737 // address, so take special care to find the storing memory operand.
5738 MachineMemOperand *MMO = nullptr;
5739 for (auto *I : MI.memoperands())
5740 if (I->isStore()) {
5741 MMO = I;
5742 break;
5743 }
5744
5745 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode))
5746 .addReg(SrcReg)
5747 .add(Base)
5748 .addImm(Disp)
5749 .addImm(CCValid)
5750 .addImm(CCMask)
5751 .addMemOperand(MMO);
5752
5753 MI.eraseFromParent();
5754 return MBB;
5755 }
5756
5757 // Get the condition needed to branch around the store.
5758 if (!Invert)
5759 CCMask ^= CCValid;
5760
5761 MachineBasicBlock *StartMBB = MBB;
5762 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB);
5763 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB);
5764
5765 // StartMBB:
5766 // BRC CCMask, JoinMBB
5767 // # fallthrough to FalseMBB
5768 MBB = StartMBB;
5769 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5770 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
5771 MBB->addSuccessor(JoinMBB);
5772 MBB->addSuccessor(FalseMBB);
5773
5774 // FalseMBB:
5775 // store %SrcReg, %Disp(%Index,%Base)
5776 // # fallthrough to JoinMBB
5777 MBB = FalseMBB;
5778 BuildMI(MBB, DL, TII->get(StoreOpcode))
5779 .addReg(SrcReg)
5780 .add(Base)
5781 .addImm(Disp)
5782 .addReg(IndexReg);
5783 MBB->addSuccessor(JoinMBB);
5784
5785 MI.eraseFromParent();
5786 return JoinMBB;
5787}
5788
5789// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_*
5790// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that
5791// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}.
5792// BitSize is the width of the field in bits, or 0 if this is a partword
5793// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize
5794// is one of the operands. Invert says whether the field should be
5795// inverted after performing BinOpcode (e.g. for NAND).
5796MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
5797 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
5798 unsigned BitSize, bool Invert) const {
5799 MachineFunction &MF = *MBB->getParent();
5800 const SystemZInstrInfo *TII =
5801 static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
5802 MachineRegisterInfo &MRI = MF.getRegInfo();
5803 bool IsSubWord = (BitSize < 32);
5804
5805 // Extract the operands. Base can be a register or a frame index.
5806 // Src2 can be a register or immediate.
5807 unsigned Dest = MI.getOperand(0).getReg();
5808 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
5809 int64_t Disp = MI.getOperand(2).getImm();
5810 MachineOperand Src2 = earlyUseOperand(MI.getOperand(3));
5811 unsigned BitShift = (IsSubWord ? MI.getOperand(4).getReg() : 0);
5812 unsigned NegBitShift = (IsSubWord ? MI.getOperand(5).getReg() : 0);
5813 DebugLoc DL = MI.getDebugLoc();
5814 if (IsSubWord)
5815 BitSize = MI.getOperand(6).getImm();
5816
5817 // Subword operations use 32-bit registers.
5818 const TargetRegisterClass *RC = (BitSize <= 32 ?
5819 &SystemZ::GR32BitRegClass :
5820 &SystemZ::GR64BitRegClass);
5821 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG;
5822 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG;
5823
5824 // Get the right opcodes for the displacement.
5825 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp);
5826 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp);
5827 assert(LOpcode && CSOpcode && "Displacement out of range")(static_cast <bool> (LOpcode && CSOpcode &&
"Displacement out of range") ? void (0) : __assert_fail ("LOpcode && CSOpcode && \"Displacement out of range\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 5827, __extension__ __PRETTY_FUNCTION__))
;
5828
5829 // Create virtual registers for temporary results.
5830 unsigned OrigVal = MRI.createVirtualRegister(RC);
5831 unsigned OldVal = MRI.createVirtualRegister(RC);
5832 unsigned NewVal = (BinOpcode || IsSubWord ?
5833 MRI.createVirtualRegister(RC) : Src2.getReg());
5834 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal);
5835 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal);
5836
5837 // Insert a basic block for the main loop.
5838 MachineBasicBlock *StartMBB = MBB;
5839 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB);
5840 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB);
5841
5842 // StartMBB:
5843 // ...
5844 // %OrigVal = L Disp(%Base)
5845 // # fall through to LoopMMB
5846 MBB = StartMBB;
5847 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
5848 MBB->addSuccessor(LoopMBB);
5849
5850 // LoopMBB:
5851 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
5852 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
5853 // %RotatedNewVal = OP %RotatedOldVal, %Src2
5854 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
5855 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
5856 // JNE LoopMBB
5857 // # fall through to DoneMMB
5858 MBB = LoopMBB;
5859 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
5860 .addReg(OrigVal).addMBB(StartMBB)
5861 .addReg(Dest).addMBB(LoopMBB);
5862 if (IsSubWord)
5863 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
5864 .addReg(OldVal).addReg(BitShift).addImm(0);
5865 if (Invert) {
5866 // Perform the operation normally and then invert every bit of the field.
5867 unsigned Tmp = MRI.createVirtualRegister(RC);
5868 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2);
5869 if (BitSize <= 32)
5870 // XILF with the upper BitSize bits set.
5871 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal)
5872 .addReg(Tmp).addImm(-1U << (32 - BitSize));
5873 else {
5874 // Use LCGR and add -1 to the result, which is more compact than
5875 // an XILF, XILH pair.
5876 unsigned Tmp2 = MRI.createVirtualRegister(RC);
5877 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp);
5878 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal)
5879 .addReg(Tmp2).addImm(-1);
5880 }
5881 } else if (BinOpcode)
5882 // A simply binary operation.
5883 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal)
5884 .addReg(RotatedOldVal)
5885 .add(Src2);
5886 else if (IsSubWord)
5887 // Use RISBG to rotate Src2 into position and use it to replace the
5888 // field in RotatedOldVal.
5889 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal)
5890 .addReg(RotatedOldVal).addReg(Src2.getReg())
5891 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize);
5892 if (IsSubWord)
5893 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
5894 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
5895 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
5896 .addReg(OldVal)
5897 .addReg(NewVal)
5898 .add(Base)
5899 .addImm(Disp);
5900 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5901 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
5902 MBB->addSuccessor(LoopMBB);
5903 MBB->addSuccessor(DoneMBB);
5904
5905 MI.eraseFromParent();
5906 return DoneMBB;
5907}
5908
5909// Implement EmitInstrWithCustomInserter for pseudo
5910// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
5911// instruction that should be used to compare the current field with the
5912// minimum or maximum value. KeepOldMask is the BRC condition-code mask
5913// for when the current field should be kept. BitSize is the width of
5914// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction.
5915MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
5916 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
5917 unsigned KeepOldMask, unsigned BitSize) const {
5918 MachineFunction &MF = *MBB->getParent();
5919 const SystemZInstrInfo *TII =
5920 static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
5921 MachineRegisterInfo &MRI = MF.getRegInfo();
5922 bool IsSubWord = (BitSize < 32);
5923
5924 // Extract the operands. Base can be a register or a frame index.
5925 unsigned Dest = MI.getOperand(0).getReg();
5926 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
5927 int64_t Disp = MI.getOperand(2).getImm();
5928 unsigned Src2 = MI.getOperand(3).getReg();
5929 unsigned BitShift = (IsSubWord ? MI.getOperand(4).getReg() : 0);
5930 unsigned NegBitShift = (IsSubWord ? MI.getOperand(5).getReg() : 0);
5931 DebugLoc DL = MI.getDebugLoc();
5932 if (IsSubWord)
5933 BitSize = MI.getOperand(6).getImm();
5934
5935 // Subword operations use 32-bit registers.
5936 const TargetRegisterClass *RC = (BitSize <= 32 ?
5937 &SystemZ::GR32BitRegClass :
5938 &SystemZ::GR64BitRegClass);
5939 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG;
5940 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG;
5941
5942 // Get the right opcodes for the displacement.
5943 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp);
5944 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp);
5945 assert(LOpcode && CSOpcode && "Displacement out of range")(static_cast <bool> (LOpcode && CSOpcode &&
"Displacement out of range") ? void (0) : __assert_fail ("LOpcode && CSOpcode && \"Displacement out of range\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 5945, __extension__ __PRETTY_FUNCTION__))
;
5946
5947 // Create virtual registers for temporary results.
5948 unsigned OrigVal = MRI.createVirtualRegister(RC);
5949 unsigned OldVal = MRI.createVirtualRegister(RC);
5950 unsigned NewVal = MRI.createVirtualRegister(RC);
5951 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal);
5952 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2);
5953 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal);
5954
5955 // Insert 3 basic blocks for the loop.
5956 MachineBasicBlock *StartMBB = MBB;
5957 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB);
5958 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB);
5959 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB);
5960 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB);
5961
5962 // StartMBB:
5963 // ...
5964 // %OrigVal = L Disp(%Base)
5965 // # fall through to LoopMMB
5966 MBB = StartMBB;
5967 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
5968 MBB->addSuccessor(LoopMBB);
5969
5970 // LoopMBB:
5971 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
5972 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
5973 // CompareOpcode %RotatedOldVal, %Src2
5974 // BRC KeepOldMask, UpdateMBB
5975 MBB = LoopMBB;
5976 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
5977 .addReg(OrigVal).addMBB(StartMBB)
5978 .addReg(Dest).addMBB(UpdateMBB);
5979 if (IsSubWord)
5980 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
5981 .addReg(OldVal).addReg(BitShift).addImm(0);
5982 BuildMI(MBB, DL, TII->get(CompareOpcode))
5983 .addReg(RotatedOldVal).addReg(Src2);
5984 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5985 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB);
5986 MBB->addSuccessor(UpdateMBB);
5987 MBB->addSuccessor(UseAltMBB);
5988
5989 // UseAltMBB:
5990 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
5991 // # fall through to UpdateMMB
5992 MBB = UseAltMBB;
5993 if (IsSubWord)
5994 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)
5995 .addReg(RotatedOldVal).addReg(Src2)
5996 .addImm(32).addImm(31 + BitSize).addImm(0);
5997 MBB->addSuccessor(UpdateMBB);
5998
5999 // UpdateMBB:
6000 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
6001 // [ %RotatedAltVal, UseAltMBB ]
6002 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
6003 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
6004 // JNE LoopMBB
6005 // # fall through to DoneMMB
6006 MBB = UpdateMBB;
6007 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)
6008 .addReg(RotatedOldVal).addMBB(LoopMBB)
6009 .addReg(RotatedAltVal).addMBB(UseAltMBB);
6010 if (IsSubWord)
6011 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
6012 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
6013 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
6014 .addReg(OldVal)
6015 .addReg(NewVal)
6016 .add(Base)
6017 .addImm(Disp);
6018 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
6019 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
6020 MBB->addSuccessor(LoopMBB);
6021 MBB->addSuccessor(DoneMBB);
6022
6023 MI.eraseFromParent();
6024 return DoneMBB;
6025}
6026
6027// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW
6028// instruction MI.
6029MachineBasicBlock *
6030SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
6031 MachineBasicBlock *MBB) const {
6032
6033 MachineFunction &MF = *MBB->getParent();
6034 const SystemZInstrInfo *TII =
6035 static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
6036 MachineRegisterInfo &MRI = MF.getRegInfo();
6037
6038 // Extract the operands. Base can be a register or a frame index.
6039 unsigned Dest = MI.getOperand(0).getReg();
6040 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
6041 int64_t Disp = MI.getOperand(2).getImm();
6042 unsigned OrigCmpVal = MI.getOperand(3).getReg();
6043 unsigned OrigSwapVal = MI.getOperand(4).getReg();
6044 unsigned BitShift = MI.getOperand(5).getReg();
6045 unsigned NegBitShift = MI.getOperand(6).getReg();
6046 int64_t BitSize = MI.getOperand(7).getImm();
6047 DebugLoc DL = MI.getDebugLoc();
6048
6049 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
6050
6051 // Get the right opcodes for the displacement.
6052 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
6053 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
6054 assert(LOpcode && CSOpcode && "Displacement out of range")(static_cast <bool> (LOpcode && CSOpcode &&
"Displacement out of range") ? void (0) : __assert_fail ("LOpcode && CSOpcode && \"Displacement out of range\""
, "/build/llvm-toolchain-snapshot-7~svn326551/lib/Target/SystemZ/SystemZISelLowering.cpp"
, 6054, __extension__ __PRETTY_FUNCTION__))
;
6055
6056 // Create virtual registers for temporary results.
6057 unsigned OrigOldVal = MRI.createVirtualRegister(RC);
6058 unsigned OldVal = MRI.createVirtualRegister(RC);
6059 unsigned CmpVal = MRI.createVirtualRegister(RC);
6060 unsigned SwapVal = MRI.createVirtualRegister(RC);
6061 unsigned StoreVal = MRI.createVirtualRegister(RC);
6062 unsigned RetryOldVal = MRI.createVirtualRegister(RC);
6063 unsigned RetryCmpVal = MRI.createVirtualRegister(RC);
6064 unsigned RetrySwapVal = MRI.createVirtualRegister(RC);
6065
6066 // Insert 2 basic blocks for the loop.
6067 MachineBasicBlock *StartMBB = MBB;
6068 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB);
6069 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB);
6070 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB);
6071
6072 // StartMBB:
6073 // ...
6074 // %OrigOldVal = L Disp(%Base)
6075 // # fall through to LoopMMB
6076 MBB = StartMBB;
6077 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal)
6078 .add(Base)
6079 .addImm(Disp)
6080 .addReg(0);
6081 MBB->addSuccessor(LoopMBB);
6082
6083 // LoopMBB:
6084 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
6085 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ]
6086 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
6087 // %Dest = RLL %OldVal, BitSize(%BitShift)
6088 // ^^ The low BitSize bits contain the field
6089 // of interest.
6090 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0
6091 // ^^ Replace the upper 32-BitSize bits of the
6092 // comparison value with those that we loaded,
6093 // so that we can use a full word comparison.
6094 // CR %Dest, %RetryCmpVal
6095 // JNE DoneMBB
6096 // # Fall through to SetMBB
6097 MBB = LoopMBB;
6098 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
6099 .addReg(OrigOldVal).addMBB(StartMBB)
6100 .addReg(RetryOldVal).addMBB(SetMBB);
6101 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal)
6102 .addReg(OrigCmpVal).addMBB(StartMBB)
6103 .addReg(RetryCmpVal).addMBB(SetMBB);
6104 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
6105 .addReg(OrigSwapVal).addMBB(StartMBB)
6106 .addReg(RetrySwapVal).addMBB(SetMBB);
6107 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest)
6108 .addReg(OldVal).addReg(BitShift).addImm(BitSize);
6109 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal)
6110 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
6111 BuildMI(MBB, DL, TII->get(SystemZ::CR))
6112 .addReg(Dest).addReg(RetryCmpVal);
6113 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
6114 .addImm(SystemZ::CCMASK_ICMP)
6115 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
6116 MBB->addSuccessor(DoneMBB);
6117 MBB->addSuccessor(SetMBB);
6118
6119 // SetMBB:
6120 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0
6121 // ^^ Replace the upper 32-BitSize bits of the new
6122 // value with those that we loaded.
6123 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
6124 // ^^ Rotate the new field to its proper position.
6125 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base)
6126 // JNE LoopMBB
6127 // # fall through to ExitMMB
6128 MBB = SetMBB;
6129 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
6130 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
6131 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
6132 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
6133 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
6134 .addReg(OldVal)
6135 .addReg(StoreVal)
6136 .add(Base)
6137 .addImm(Disp);
6138 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
6139 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
6140 MBB->addSuccessor(LoopMBB);
6141 MBB->addSuccessor(DoneMBB);
6142
6143 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
6144 // to the block after the loop. At this point, CC may have been defined
6145 // either by the CR in LoopMBB or by the CS in SetMBB.
6146 if (!MI.registerDefIsDead(SystemZ::CC))
6147 DoneMBB->addLiveIn(SystemZ::CC);
6148
6149 MI.eraseFromParent();
6150 return DoneMBB;
6151}
6152
6153// Emit a move from two GR64s to a GR128.
6154MachineBasicBlock *
6155SystemZTargetLowering::emitPair128(MachineInstr &MI,
6156 MachineBasicBlock *MBB) const {
6157 MachineFunction &MF = *MBB->getParent();
6158 const SystemZInstrInfo *TII =
6159 static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
6160 MachineRegisterInfo &MRI = MF.getRegInfo();
6161 DebugLoc DL = MI.getDebugLoc();
6162
6163 unsigned Dest = MI.getOperand(0).getReg();
6164 unsigned Hi = MI.getOperand(1).getReg();
6165 unsigned Lo = MI.getOperand(2).getReg();
6166 unsigned Tmp1 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
6167 unsigned Tmp2 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
6168
6169 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Tmp1);
6170