LLVM 17.0.0git
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the SystemZ target.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SystemZISelLowering.h"
17#include "llvm/Support/Debug.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
25
26namespace {
27// Used to build addressing modes.
28struct SystemZAddressingMode {
29 // The shape of the address.
30 enum AddrForm {
31 // base+displacement
32 FormBD,
33
34 // base+displacement+index for load and store operands
35 FormBDXNormal,
36
37 // base+displacement+index for load address operands
38 FormBDXLA,
39
40 // base+displacement+index+ADJDYNALLOC
41 FormBDXDynAlloc
42 };
43 AddrForm Form;
44
45 // The type of displacement. The enum names here correspond directly
46 // to the definitions in SystemZOperand.td. We could split them into
47 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
48 enum DispRange {
49 Disp12Only,
50 Disp12Pair,
51 Disp20Only,
52 Disp20Only128,
53 Disp20Pair
54 };
55 DispRange DR;
56
57 // The parts of the address. The address is equivalent to:
58 //
59 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
61 int64_t Disp;
63 bool IncludesDynAlloc;
64
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
67
68 // True if the address can have an index register.
69 bool hasIndexField() { return Form != FormBD; }
70
71 // True if the address can (and must) include ADJDYNALLOC.
72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
74 void dump(const llvm::SelectionDAG *DAG) {
75 errs() << "SystemZAddressingMode " << this << '\n';
76
77 errs() << " Base ";
78 if (Base.getNode())
79 Base.getNode()->dump(DAG);
80 else
81 errs() << "null\n";
82
83 if (hasIndexField()) {
84 errs() << " Index ";
85 if (Index.getNode())
86 Index.getNode()->dump(DAG);
87 else
88 errs() << "null\n";
89 }
90
91 errs() << " Disp " << Disp;
92 if (IncludesDynAlloc)
93 errs() << " + ADJDYNALLOC";
94 errs() << '\n';
95 }
96};
97
98// Return a mask with Count low bits set.
99static uint64_t allOnes(unsigned int Count) {
100 assert(Count <= 64);
101 if (Count > 63)
102 return UINT64_MAX;
103 return (uint64_t(1) << Count) - 1;
104}
105
106// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107// given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108// Rotate (I5). The combined operand value is effectively:
109//
110// (or (rotl Input, Rotate), ~Mask)
111//
112// for RNSBG and:
113//
114// (and (rotl Input, Rotate), Mask)
115//
116// otherwise. The output value has BitSize bits, although Input may be
117// narrower (in which case the upper bits are don't care), or wider (in which
118// case the result will be truncated as part of the operation).
119struct RxSBGOperands {
120 RxSBGOperands(unsigned Op, SDValue N)
121 : Opcode(Op), BitSize(N.getValueSizeInBits()),
122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123 Rotate(0) {}
124
125 unsigned Opcode;
126 unsigned BitSize;
127 uint64_t Mask;
128 SDValue Input;
129 unsigned Start;
130 unsigned End;
131 unsigned Rotate;
132};
133
134class SystemZDAGToDAGISel : public SelectionDAGISel {
135 const SystemZSubtarget *Subtarget;
136
137 // Used by SystemZOperands.td to create integer constants.
138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140 }
141
142 const SystemZTargetMachine &getTargetMachine() const {
143 return static_cast<const SystemZTargetMachine &>(TM);
144 }
145
146 const SystemZInstrInfo *getInstrInfo() const {
147 return Subtarget->getInstrInfo();
148 }
149
150 // Try to fold more of the base or index of AM into AM, where IsBase
151 // selects between the base and index.
152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154 // Try to describe N in AM, returning true on success.
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157 // Extract individual target operands from matched address AM.
158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159 SDValue &Base, SDValue &Disp) const;
160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163 // Try to match Addr as a FormBD address with displacement type DR.
164 // Return true on success, storing the base and displacement in
165 // Base and Disp respectively.
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
168
169 // Try to match Addr as a FormBDX address with displacement type DR.
170 // Return true on success and if the result had no index. Store the
171 // base and displacement in Base and Disp respectively.
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
174
175 // Try to match Addr as a FormBDX* address of form Form with
176 // displacement type DR. Return true on success, storing the base,
177 // displacement and index in Base, Disp and Index respectively.
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182 // PC-relative address matching routines used by SystemZOperands.td.
183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184 if (SystemZISD::isPCREL(Addr.getOpcode())) {
185 Target = Addr.getOperand(0);
186 return true;
187 }
188 return false;
189 }
190
191 // BD matching routines used by SystemZOperands.td.
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194 }
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197 }
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200 }
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203 }
204
205 // MVI matching routines used by SystemZOperands.td.
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208 }
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211 }
212
213 // BDX matching routines used by SystemZOperands.td.
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
218 Addr, Base, Disp, Index);
219 }
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
224 Addr, Base, Disp, Index);
225 }
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
230 Addr, Base, Disp, Index);
231 }
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
236 Addr, Base, Disp, Index);
237 }
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
242 Addr, Base, Disp, Index);
243 }
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
248 Addr, Base, Disp, Index);
249 }
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
254 Addr, Base, Disp, Index);
255 }
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
260 Addr, Base, Disp, Index);
261 }
262
263 // Try to match Addr as an address with a base, 12-bit displacement
264 // and index, where the index is element Elem of a vector.
265 // Return true on success, storing the base, displacement and vector
266 // in Base, Disp and Index respectively.
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
269
270 // Check whether (or Op (and X InsertMask)) is effectively an insertion
271 // of X into bits InsertMask of some Y != Op. Return true if so and
272 // set Op to that Y.
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276 // Return true on success.
277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279 // Try to fold some of RxSBG.Input into other fields of RxSBG.
280 // Return true on success.
281 bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283 // Return an undefined value of type VT.
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286 // Convert N to VT, if it isn't already.
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289 // Try to implement AND or shift node N using RISBG with the zero flag set.
290 // Return the selected node on success, otherwise return null.
291 bool tryRISBGZero(SDNode *N);
292
293 // Try to use RISBG or Opcode to implement OR or XOR node N.
294 // Return the selected node on success, otherwise return null.
295 bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297 // If Op0 is null, then Node is a constant that can be loaded using:
298 //
299 // (Opcode UpperVal LowerVal)
300 //
301 // If Op0 is nonnull, then Node can be implemented using:
302 //
303 // (Opcode (Opcode Op0 UpperVal) LowerVal)
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305 uint64_t UpperVal, uint64_t LowerVal);
306
307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
308 SDNode *Node);
309
310 // Try to use gather instruction Opcode to implement vector insertion N.
311 bool tryGather(SDNode *N, unsigned Opcode);
312
313 // Try to use scatter instruction Opcode to implement store Store.
314 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
315
316 // Change a chain of {load; op; store} of the same value into a simple op
317 // through memory of that value, if the uses of the modified value and its
318 // address are suitable.
319 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
320
321 // Return true if Load and Store are loads and stores of the same size
322 // and are guaranteed not to overlap. Such operations can be implemented
323 // using block (SS-format) instructions.
324 //
325 // Partial overlap would lead to incorrect code, since the block operations
326 // are logically bytewise, even though they have a fast path for the
327 // non-overlapping case. We also need to avoid full overlap (i.e. two
328 // addresses that might be equal at run time) because although that case
329 // would be handled correctly, it might be implemented by millicode.
330 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
331
332 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
333 // from Y to X.
334 bool storeLoadCanUseMVC(SDNode *N) const;
335
336 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
337 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
338 // to X.
339 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
340
341 // Return true if N (a load or a store) fullfills the alignment
342 // requirements for a PC-relative access.
343 bool storeLoadIsAligned(SDNode *N) const;
344
345 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
346 SDValue expandSelectBoolean(SDNode *Node);
347
348public:
349 static char ID;
350
351 SystemZDAGToDAGISel() = delete;
352
353 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
354 : SelectionDAGISel(ID, TM, OptLevel) {}
355
356 bool runOnMachineFunction(MachineFunction &MF) override {
357 const Function &F = MF.getFunction();
358 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
359 if (F.hasFnAttribute("mnop-mcount"))
360 report_fatal_error("mnop-mcount only supported with fentry-call");
361 if (F.hasFnAttribute("mrecord-mcount"))
362 report_fatal_error("mrecord-mcount only supported with fentry-call");
363 }
364
365 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
367 }
368
369 // Override SelectionDAGISel.
370 void Select(SDNode *Node) override;
371 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
372 std::vector<SDValue> &OutOps) override;
373 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
374 void PreprocessISelDAG() override;
375
376 // Include the pieces autogenerated from the target description.
377 #include "SystemZGenDAGISel.inc"
378};
379} // end anonymous namespace
380
381char SystemZDAGToDAGISel::ID = 0;
382
383INITIALIZE_PASS(SystemZDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false)
384
386 CodeGenOpt::Level OptLevel) {
387 return new SystemZDAGToDAGISel(TM, OptLevel);
388}
389
390// Return true if Val should be selected as a displacement for an address
391// with range DR. Here we're interested in the range of both the instruction
392// described by DR and of any pairing instruction.
393static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
394 switch (DR) {
395 case SystemZAddressingMode::Disp12Only:
396 return isUInt<12>(Val);
397
398 case SystemZAddressingMode::Disp12Pair:
399 case SystemZAddressingMode::Disp20Only:
400 case SystemZAddressingMode::Disp20Pair:
401 return isInt<20>(Val);
402
403 case SystemZAddressingMode::Disp20Only128:
404 return isInt<20>(Val) && isInt<20>(Val + 8);
405 }
406 llvm_unreachable("Unhandled displacement range");
407}
408
409// Change the base or index in AM to Value, where IsBase selects
410// between the base and index.
411static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
412 SDValue Value) {
413 if (IsBase)
414 AM.Base = Value;
415 else
416 AM.Index = Value;
417}
418
419// The base or index of AM is equivalent to Value + ADJDYNALLOC,
420// where IsBase selects between the base and index. Try to fold the
421// ADJDYNALLOC into AM.
422static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
423 SDValue Value) {
424 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
425 changeComponent(AM, IsBase, Value);
426 AM.IncludesDynAlloc = true;
427 return true;
428 }
429 return false;
430}
431
432// The base of AM is equivalent to Base + Index. Try to use Index as
433// the index register.
434static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
435 SDValue Index) {
436 if (AM.hasIndexField() && !AM.Index.getNode()) {
437 AM.Base = Base;
438 AM.Index = Index;
439 return true;
440 }
441 return false;
442}
443
444// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
445// between the base and index. Try to fold Op1 into AM's displacement.
446static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
447 SDValue Op0, uint64_t Op1) {
448 // First try adjusting the displacement.
449 int64_t TestDisp = AM.Disp + Op1;
450 if (selectDisp(AM.DR, TestDisp)) {
451 changeComponent(AM, IsBase, Op0);
452 AM.Disp = TestDisp;
453 return true;
454 }
455
456 // We could consider forcing the displacement into a register and
457 // using it as an index, but it would need to be carefully tuned.
458 return false;
459}
460
461bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
462 bool IsBase) const {
463 SDValue N = IsBase ? AM.Base : AM.Index;
464 unsigned Opcode = N.getOpcode();
465 if (Opcode == ISD::TRUNCATE) {
466 N = N.getOperand(0);
467 Opcode = N.getOpcode();
468 }
469 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
470 SDValue Op0 = N.getOperand(0);
471 SDValue Op1 = N.getOperand(1);
472
473 unsigned Op0Code = Op0->getOpcode();
474 unsigned Op1Code = Op1->getOpcode();
475
476 if (Op0Code == SystemZISD::ADJDYNALLOC)
477 return expandAdjDynAlloc(AM, IsBase, Op1);
478 if (Op1Code == SystemZISD::ADJDYNALLOC)
479 return expandAdjDynAlloc(AM, IsBase, Op0);
480
481 if (Op0Code == ISD::Constant)
482 return expandDisp(AM, IsBase, Op1,
483 cast<ConstantSDNode>(Op0)->getSExtValue());
484 if (Op1Code == ISD::Constant)
485 return expandDisp(AM, IsBase, Op0,
486 cast<ConstantSDNode>(Op1)->getSExtValue());
487
488 if (IsBase && expandIndex(AM, Op0, Op1))
489 return true;
490 }
491 if (Opcode == SystemZISD::PCREL_OFFSET) {
492 SDValue Full = N.getOperand(0);
493 SDValue Base = N.getOperand(1);
494 SDValue Anchor = Base.getOperand(0);
495 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
496 cast<GlobalAddressSDNode>(Anchor)->getOffset());
497 return expandDisp(AM, IsBase, Base, Offset);
498 }
499 return false;
500}
501
502// Return true if an instruction with displacement range DR should be
503// used for displacement value Val. selectDisp(DR, Val) must already hold.
504static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
505 assert(selectDisp(DR, Val) && "Invalid displacement");
506 switch (DR) {
507 case SystemZAddressingMode::Disp12Only:
508 case SystemZAddressingMode::Disp20Only:
509 case SystemZAddressingMode::Disp20Only128:
510 return true;
511
512 case SystemZAddressingMode::Disp12Pair:
513 // Use the other instruction if the displacement is too large.
514 return isUInt<12>(Val);
515
516 case SystemZAddressingMode::Disp20Pair:
517 // Use the other instruction if the displacement is small enough.
518 return !isUInt<12>(Val);
519 }
520 llvm_unreachable("Unhandled displacement range");
521}
522
523// Return true if Base + Disp + Index should be performed by LA(Y).
524static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
525 // Don't use LA(Y) for constants.
526 if (!Base)
527 return false;
528
529 // Always use LA(Y) for frame addresses, since we know that the destination
530 // register is almost always (perhaps always) going to be different from
531 // the frame register.
532 if (Base->getOpcode() == ISD::FrameIndex)
533 return true;
534
535 if (Disp) {
536 // Always use LA(Y) if there is a base, displacement and index.
537 if (Index)
538 return true;
539
540 // Always use LA if the displacement is small enough. It should always
541 // be no worse than AGHI (and better if it avoids a move).
542 if (isUInt<12>(Disp))
543 return true;
544
545 // For similar reasons, always use LAY if the constant is too big for AGHI.
546 // LAY should be no worse than AGFI.
547 if (!isInt<16>(Disp))
548 return true;
549 } else {
550 // Don't use LA for plain registers.
551 if (!Index)
552 return false;
553
554 // Don't use LA for plain addition if the index operand is only used
555 // once. It should be a natural two-operand addition in that case.
556 if (Index->hasOneUse())
557 return false;
558
559 // Prefer addition if the second operation is sign-extended, in the
560 // hope of using AGF.
561 unsigned IndexOpcode = Index->getOpcode();
562 if (IndexOpcode == ISD::SIGN_EXTEND ||
563 IndexOpcode == ISD::SIGN_EXTEND_INREG)
564 return false;
565 }
566
567 // Don't use LA for two-operand addition if either operand is only
568 // used once. The addition instructions are better in that case.
569 if (Base->hasOneUse())
570 return false;
571
572 return true;
573}
574
575// Return true if Addr is suitable for AM, updating AM if so.
576bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
577 SystemZAddressingMode &AM) const {
578 // Start out assuming that the address will need to be loaded separately,
579 // then try to extend it as much as we can.
580 AM.Base = Addr;
581
582 // First try treating the address as a constant.
583 if (Addr.getOpcode() == ISD::Constant &&
584 expandDisp(AM, true, SDValue(),
585 cast<ConstantSDNode>(Addr)->getSExtValue()))
586 ;
587 // Also see if it's a bare ADJDYNALLOC.
588 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
589 expandAdjDynAlloc(AM, true, SDValue()))
590 ;
591 else
592 // Otherwise try expanding each component.
593 while (expandAddress(AM, true) ||
594 (AM.Index.getNode() && expandAddress(AM, false)))
595 continue;
596
597 // Reject cases where it isn't profitable to use LA(Y).
598 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
599 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
600 return false;
601
602 // Reject cases where the other instruction in a pair should be used.
603 if (!isValidDisp(AM.DR, AM.Disp))
604 return false;
605
606 // Make sure that ADJDYNALLOC is included where necessary.
607 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
608 return false;
609
610 LLVM_DEBUG(AM.dump(CurDAG));
611 return true;
612}
613
614// Insert a node into the DAG at least before Pos. This will reposition
615// the node as needed, and will assign it a node ID that is <= Pos's ID.
616// Note that this does *not* preserve the uniqueness of node IDs!
617// The selection DAG must no longer depend on their uniqueness when this
618// function is used.
619static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
620 if (N->getNodeId() == -1 ||
623 DAG->RepositionNode(Pos->getIterator(), N.getNode());
624 // Mark Node as invalid for pruning as after this it may be a successor to a
625 // selected node but otherwise be in the same position of Pos.
626 // Conservatively mark it with the same -abs(Id) to assure node id
627 // invariant is preserved.
628 N->setNodeId(Pos->getNodeId());
630 }
631}
632
633void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
634 EVT VT, SDValue &Base,
635 SDValue &Disp) const {
636 Base = AM.Base;
637 if (!Base.getNode())
638 // Register 0 means "no base". This is mostly useful for shifts.
639 Base = CurDAG->getRegister(0, VT);
640 else if (Base.getOpcode() == ISD::FrameIndex) {
641 // Lower a FrameIndex to a TargetFrameIndex.
642 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
643 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
644 } else if (Base.getValueType() != VT) {
645 // Truncate values from i64 to i32, for shifts.
646 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
647 "Unexpected truncation");
648 SDLoc DL(Base);
649 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
650 insertDAGNode(CurDAG, Base.getNode(), Trunc);
651 Base = Trunc;
652 }
653
654 // Lower the displacement to a TargetConstant.
655 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
656}
657
658void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
659 EVT VT, SDValue &Base,
660 SDValue &Disp,
661 SDValue &Index) const {
662 getAddressOperands(AM, VT, Base, Disp);
663
664 Index = AM.Index;
665 if (!Index.getNode())
666 // Register 0 means "no index".
667 Index = CurDAG->getRegister(0, VT);
668}
669
670bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
672 SDValue &Disp) const {
673 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
674 if (!selectAddress(Addr, AM))
675 return false;
676
677 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
678 return true;
679}
680
681bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
683 SDValue &Disp) const {
684 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
685 if (!selectAddress(Addr, AM) || AM.Index.getNode())
686 return false;
687
688 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
689 return true;
690}
691
692bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
693 SystemZAddressingMode::DispRange DR,
695 SDValue &Disp, SDValue &Index) const {
696 SystemZAddressingMode AM(Form, DR);
697 if (!selectAddress(Addr, AM))
698 return false;
699
700 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
701 return true;
702}
703
704bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
705 SDValue &Base,
706 SDValue &Disp,
707 SDValue &Index) const {
708 SDValue Regs[2];
709 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
710 Regs[0].getNode() && Regs[1].getNode()) {
711 for (unsigned int I = 0; I < 2; ++I) {
712 Base = Regs[I];
713 Index = Regs[1 - I];
714 // We can't tell here whether the index vector has the right type
715 // for the access; the caller needs to do that instead.
716 if (Index.getOpcode() == ISD::ZERO_EXTEND)
718 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
719 Index.getOperand(1) == Elem) {
721 return true;
722 }
723 }
724 }
725 return false;
726}
727
728bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
729 uint64_t InsertMask) const {
730 // We're only interested in cases where the insertion is into some operand
731 // of Op, rather than into Op itself. The only useful case is an AND.
732 if (Op.getOpcode() != ISD::AND)
733 return false;
734
735 // We need a constant mask.
736 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
737 if (!MaskNode)
738 return false;
739
740 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
741 uint64_t AndMask = MaskNode->getZExtValue();
742 if (InsertMask & AndMask)
743 return false;
744
745 // It's only an insertion if all bits are covered or are known to be zero.
746 // The inner check covers all cases but is more expensive.
747 uint64_t Used = allOnes(Op.getValueSizeInBits());
748 if (Used != (AndMask | InsertMask)) {
749 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
750 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
751 return false;
752 }
753
754 Op = Op.getOperand(0);
755 return true;
756}
757
758bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
759 uint64_t Mask) const {
760 const SystemZInstrInfo *TII = getInstrInfo();
761 if (RxSBG.Rotate != 0)
762 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
763 Mask &= RxSBG.Mask;
764 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
765 RxSBG.Mask = Mask;
766 return true;
767 }
768 return false;
769}
770
771// Return true if any bits of (RxSBG.Input & Mask) are significant.
772static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
773 // Rotate the mask in the same way as RxSBG.Input is rotated.
774 if (RxSBG.Rotate != 0)
775 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
776 return (Mask & RxSBG.Mask) != 0;
777}
778
779bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
780 SDValue N = RxSBG.Input;
781 unsigned Opcode = N.getOpcode();
782 switch (Opcode) {
783 case ISD::TRUNCATE: {
784 if (RxSBG.Opcode == SystemZ::RNSBG)
785 return false;
786 uint64_t BitSize = N.getValueSizeInBits();
787 uint64_t Mask = allOnes(BitSize);
788 if (!refineRxSBGMask(RxSBG, Mask))
789 return false;
790 RxSBG.Input = N.getOperand(0);
791 return true;
792 }
793 case ISD::AND: {
794 if (RxSBG.Opcode == SystemZ::RNSBG)
795 return false;
796
797 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
798 if (!MaskNode)
799 return false;
800
801 SDValue Input = N.getOperand(0);
802 uint64_t Mask = MaskNode->getZExtValue();
803 if (!refineRxSBGMask(RxSBG, Mask)) {
804 // If some bits of Input are already known zeros, those bits will have
805 // been removed from the mask. See if adding them back in makes the
806 // mask suitable.
807 KnownBits Known = CurDAG->computeKnownBits(Input);
808 Mask |= Known.Zero.getZExtValue();
809 if (!refineRxSBGMask(RxSBG, Mask))
810 return false;
811 }
812 RxSBG.Input = Input;
813 return true;
814 }
815
816 case ISD::OR: {
817 if (RxSBG.Opcode != SystemZ::RNSBG)
818 return false;
819
820 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
821 if (!MaskNode)
822 return false;
823
824 SDValue Input = N.getOperand(0);
825 uint64_t Mask = ~MaskNode->getZExtValue();
826 if (!refineRxSBGMask(RxSBG, Mask)) {
827 // If some bits of Input are already known ones, those bits will have
828 // been removed from the mask. See if adding them back in makes the
829 // mask suitable.
830 KnownBits Known = CurDAG->computeKnownBits(Input);
831 Mask &= ~Known.One.getZExtValue();
832 if (!refineRxSBGMask(RxSBG, Mask))
833 return false;
834 }
835 RxSBG.Input = Input;
836 return true;
837 }
838
839 case ISD::ROTL: {
840 // Any 64-bit rotate left can be merged into the RxSBG.
841 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
842 return false;
843 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
844 if (!CountNode)
845 return false;
846
847 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
848 RxSBG.Input = N.getOperand(0);
849 return true;
850 }
851
852 case ISD::ANY_EXTEND:
853 // Bits above the extended operand are don't-care.
854 RxSBG.Input = N.getOperand(0);
855 return true;
856
857 case ISD::ZERO_EXTEND:
858 if (RxSBG.Opcode != SystemZ::RNSBG) {
859 // Restrict the mask to the extended operand.
860 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
861 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
862 return false;
863
864 RxSBG.Input = N.getOperand(0);
865 return true;
866 }
867 [[fallthrough]];
868
869 case ISD::SIGN_EXTEND: {
870 // Check that the extension bits are don't-care (i.e. are masked out
871 // by the final mask).
872 unsigned BitSize = N.getValueSizeInBits();
873 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
874 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
875 // In the case where only the sign bit is active, increase Rotate with
876 // the extension width.
877 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
878 RxSBG.Rotate += (BitSize - InnerBitSize);
879 else
880 return false;
881 }
882
883 RxSBG.Input = N.getOperand(0);
884 return true;
885 }
886
887 case ISD::SHL: {
888 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
889 if (!CountNode)
890 return false;
891
892 uint64_t Count = CountNode->getZExtValue();
893 unsigned BitSize = N.getValueSizeInBits();
894 if (Count < 1 || Count >= BitSize)
895 return false;
896
897 if (RxSBG.Opcode == SystemZ::RNSBG) {
898 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
899 // count bits from RxSBG.Input are ignored.
900 if (maskMatters(RxSBG, allOnes(Count)))
901 return false;
902 } else {
903 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
904 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
905 return false;
906 }
907
908 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
909 RxSBG.Input = N.getOperand(0);
910 return true;
911 }
912
913 case ISD::SRL:
914 case ISD::SRA: {
915 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
916 if (!CountNode)
917 return false;
918
919 uint64_t Count = CountNode->getZExtValue();
920 unsigned BitSize = N.getValueSizeInBits();
921 if (Count < 1 || Count >= BitSize)
922 return false;
923
924 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
925 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
926 // count bits from RxSBG.Input are ignored.
927 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
928 return false;
929 } else {
930 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
931 // which is similar to SLL above.
932 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
933 return false;
934 }
935
936 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
937 RxSBG.Input = N.getOperand(0);
938 return true;
939 }
940 default:
941 return false;
942 }
943}
944
945SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
946 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
947 return SDValue(N, 0);
948}
949
950SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
951 SDValue N) const {
952 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
953 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
954 DL, VT, getUNDEF(DL, MVT::i64), N);
955 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
956 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
957 assert(N.getValueType() == VT && "Unexpected value types");
958 return N;
959}
960
961bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
962 SDLoc DL(N);
963 EVT VT = N->getValueType(0);
964 if (!VT.isInteger() || VT.getSizeInBits() > 64)
965 return false;
966 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
967 unsigned Count = 0;
968 while (expandRxSBG(RISBG))
969 // The widening or narrowing is expected to be free.
970 // Counting widening or narrowing as a saved operation will result in
971 // preferring an R*SBG over a simple shift/logical instruction.
972 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
973 RISBG.Input.getOpcode() != ISD::TRUNCATE)
974 Count += 1;
975 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
976 return false;
977
978 // Prefer to use normal shift instructions over RISBG, since they can handle
979 // all cases and are sometimes shorter.
980 if (Count == 1 && N->getOpcode() != ISD::AND)
981 return false;
982
983 // Prefer register extensions like LLC over RISBG. Also prefer to start
984 // out with normal ANDs if one instruction would be enough. We can convert
985 // these ANDs into an RISBG later if a three-address instruction is useful.
986 if (RISBG.Rotate == 0) {
987 bool PreferAnd = false;
988 // Prefer AND for any 32-bit and-immediate operation.
989 if (VT == MVT::i32)
990 PreferAnd = true;
991 // As well as for any 64-bit operation that can be implemented via LLC(R),
992 // LLH(R), LLGT(R), or one of the and-immediate instructions.
993 else if (RISBG.Mask == 0xff ||
994 RISBG.Mask == 0xffff ||
995 RISBG.Mask == 0x7fffffff ||
996 SystemZ::isImmLF(~RISBG.Mask) ||
997 SystemZ::isImmHF(~RISBG.Mask))
998 PreferAnd = true;
999 // And likewise for the LLZRGF instruction, which doesn't have a register
1000 // to register version.
1001 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1002 if (Load->getMemoryVT() == MVT::i32 &&
1003 (Load->getExtensionType() == ISD::EXTLOAD ||
1004 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1005 RISBG.Mask == 0xffffff00 &&
1006 Subtarget->hasLoadAndZeroRightmostByte())
1007 PreferAnd = true;
1008 }
1009 if (PreferAnd) {
1010 // Replace the current node with an AND. Note that the current node
1011 // might already be that same AND, in which case it is already CSE'd
1012 // with it, and we must not call ReplaceNode.
1013 SDValue In = convertTo(DL, VT, RISBG.Input);
1014 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1015 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1016 if (N != New.getNode()) {
1017 insertDAGNode(CurDAG, N, Mask);
1018 insertDAGNode(CurDAG, N, New);
1019 ReplaceNode(N, New.getNode());
1020 N = New.getNode();
1021 }
1022 // Now, select the machine opcode to implement this operation.
1023 if (!N->isMachineOpcode())
1024 SelectCode(N);
1025 return true;
1026 }
1027 }
1028
1029 unsigned Opcode = SystemZ::RISBG;
1030 // Prefer RISBGN if available, since it does not clobber CC.
1031 if (Subtarget->hasMiscellaneousExtensions())
1032 Opcode = SystemZ::RISBGN;
1033 EVT OpcodeVT = MVT::i64;
1034 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1035 // We can only use the 32-bit instructions if all source bits are
1036 // in the low 32 bits without wrapping, both after rotation (because
1037 // of the smaller range for Start and End) and before rotation
1038 // (because the input value is truncated).
1039 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1040 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1041 ((RISBG.End + RISBG.Rotate) & 63) >=
1042 ((RISBG.Start + RISBG.Rotate) & 63)) {
1043 Opcode = SystemZ::RISBMux;
1044 OpcodeVT = MVT::i32;
1045 RISBG.Start &= 31;
1046 RISBG.End &= 31;
1047 }
1048 SDValue Ops[5] = {
1049 getUNDEF(DL, OpcodeVT),
1050 convertTo(DL, OpcodeVT, RISBG.Input),
1051 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1052 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1053 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1054 };
1055 SDValue New = convertTo(
1056 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1057 ReplaceNode(N, New.getNode());
1058 return true;
1059}
1060
1061bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1062 SDLoc DL(N);
1063 EVT VT = N->getValueType(0);
1064 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1065 return false;
1066 // Try treating each operand of N as the second operand of the RxSBG
1067 // and see which goes deepest.
1068 RxSBGOperands RxSBG[] = {
1069 RxSBGOperands(Opcode, N->getOperand(0)),
1070 RxSBGOperands(Opcode, N->getOperand(1))
1071 };
1072 unsigned Count[] = { 0, 0 };
1073 for (unsigned I = 0; I < 2; ++I)
1074 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I]))
1075 // In cases of multiple users it seems better to keep the simple
1076 // instruction as they are one cycle faster, and it also helps in cases
1077 // where both inputs share a common node.
1078 // The widening or narrowing is expected to be free. Counting widening
1079 // or narrowing as a saved operation will result in preferring an R*SBG
1080 // over a simple shift/logical instruction.
1081 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1082 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1083 Count[I] += 1;
1084
1085 // Do nothing if neither operand is suitable.
1086 if (Count[0] == 0 && Count[1] == 0)
1087 return false;
1088
1089 // Pick the deepest second operand.
1090 unsigned I = Count[0] > Count[1] ? 0 : 1;
1091 SDValue Op0 = N->getOperand(I ^ 1);
1092
1093 // Prefer IC for character insertions from memory.
1094 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1095 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1096 if (Load->getMemoryVT() == MVT::i8)
1097 return false;
1098
1099 // See whether we can avoid an AND in the first operand by converting
1100 // ROSBG to RISBG.
1101 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1102 Opcode = SystemZ::RISBG;
1103 // Prefer RISBGN if available, since it does not clobber CC.
1104 if (Subtarget->hasMiscellaneousExtensions())
1105 Opcode = SystemZ::RISBGN;
1106 }
1107
1108 SDValue Ops[5] = {
1109 convertTo(DL, MVT::i64, Op0),
1110 convertTo(DL, MVT::i64, RxSBG[I].Input),
1111 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1112 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1113 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1114 };
1115 SDValue New = convertTo(
1116 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1117 ReplaceNode(N, New.getNode());
1118 return true;
1119}
1120
1121void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1122 SDValue Op0, uint64_t UpperVal,
1123 uint64_t LowerVal) {
1124 EVT VT = Node->getValueType(0);
1125 SDLoc DL(Node);
1126 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1127 if (Op0.getNode())
1128 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1129
1130 {
1131 // When we haven't passed in Op0, Upper will be a constant. In order to
1132 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1133 // SelectCode first and end up with an opaque machine node. This means that
1134 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1135 // SelectCode.
1136 //
1137 // Note that in the case where Op0 is passed in we could just call
1138 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1139 // the handle at all, but it's fine to do it here.
1140 //
1141 // TODO: This is a pretty hacky way to do this. Can we do something that
1142 // doesn't require a two paragraph explanation?
1143 HandleSDNode Handle(Upper);
1144 SelectCode(Upper.getNode());
1145 Upper = Handle.getValue();
1146 }
1147
1148 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1149 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1150
1151 ReplaceNode(Node, Or.getNode());
1152
1153 SelectCode(Or.getNode());
1154}
1155
1156void SystemZDAGToDAGISel::loadVectorConstant(
1157 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1161 "Bad opcode!");
1162 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1163 EVT VT = Node->getValueType(0);
1164 SDLoc DL(Node);
1166 for (unsigned OpVal : VCI.OpVals)
1167 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1168 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1169
1170 if (VCI.VecVT == VT.getSimpleVT())
1171 ReplaceNode(Node, Op.getNode());
1172 else if (VT.getSizeInBits() == 128) {
1173 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1174 ReplaceNode(Node, BitCast.getNode());
1175 SelectCode(BitCast.getNode());
1176 } else { // float or double
1177 unsigned SubRegIdx =
1178 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1179 ReplaceNode(
1180 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1181 }
1182 SelectCode(Op.getNode());
1183}
1184
1185bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1186 SDValue ElemV = N->getOperand(2);
1187 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1188 if (!ElemN)
1189 return false;
1190
1191 unsigned Elem = ElemN->getZExtValue();
1192 EVT VT = N->getValueType(0);
1193 if (Elem >= VT.getVectorNumElements())
1194 return false;
1195
1196 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1197 if (!Load || !Load->hasNUsesOfValue(1, 0))
1198 return false;
1199 if (Load->getMemoryVT().getSizeInBits() !=
1200 Load->getValueType(0).getSizeInBits())
1201 return false;
1202
1203 SDValue Base, Disp, Index;
1204 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1205 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1206 return false;
1207
1208 SDLoc DL(Load);
1209 SDValue Ops[] = {
1210 N->getOperand(0), Base, Disp, Index,
1211 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1212 };
1213 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1214 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1215 ReplaceNode(N, Res);
1216 return true;
1217}
1218
1219bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1220 SDValue Value = Store->getValue();
1221 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1222 return false;
1223 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1224 return false;
1225
1226 SDValue ElemV = Value.getOperand(1);
1227 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1228 if (!ElemN)
1229 return false;
1230
1231 SDValue Vec = Value.getOperand(0);
1232 EVT VT = Vec.getValueType();
1233 unsigned Elem = ElemN->getZExtValue();
1234 if (Elem >= VT.getVectorNumElements())
1235 return false;
1236
1237 SDValue Base, Disp, Index;
1238 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1239 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1240 return false;
1241
1242 SDLoc DL(Store);
1243 SDValue Ops[] = {
1244 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1245 Store->getChain()
1246 };
1247 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1248 return true;
1249}
1250
1251// Check whether or not the chain ending in StoreNode is suitable for doing
1252// the {load; op; store} to modify transformation.
1254 SDValue StoredVal, SelectionDAG *CurDAG,
1255 LoadSDNode *&LoadNode,
1256 SDValue &InputChain) {
1257 // Is the stored value result 0 of the operation?
1258 if (StoredVal.getResNo() != 0)
1259 return false;
1260
1261 // Are there other uses of the loaded value than the operation?
1262 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1263 return false;
1264
1265 // Is the store non-extending and non-indexed?
1266 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1267 return false;
1268
1269 SDValue Load = StoredVal->getOperand(0);
1270 // Is the stored value a non-extending and non-indexed load?
1271 if (!ISD::isNormalLoad(Load.getNode()))
1272 return false;
1273
1274 // Return LoadNode by reference.
1275 LoadNode = cast<LoadSDNode>(Load);
1276
1277 // Is store the only read of the loaded value?
1278 if (!Load.hasOneUse())
1279 return false;
1280
1281 // Is the address of the store the same as the load?
1282 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1283 LoadNode->getOffset() != StoreNode->getOffset())
1284 return false;
1285
1286 // Check if the chain is produced by the load or is a TokenFactor with
1287 // the load output chain as an operand. Return InputChain by reference.
1288 SDValue Chain = StoreNode->getChain();
1289
1290 bool ChainCheck = false;
1291 if (Chain == Load.getValue(1)) {
1292 ChainCheck = true;
1293 InputChain = LoadNode->getChain();
1294 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1295 SmallVector<SDValue, 4> ChainOps;
1296 SmallVector<const SDNode *, 4> LoopWorklist;
1298 const unsigned int Max = 1024;
1299 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1300 SDValue Op = Chain.getOperand(i);
1301 if (Op == Load.getValue(1)) {
1302 ChainCheck = true;
1303 // Drop Load, but keep its chain. No cycle check necessary.
1304 ChainOps.push_back(Load.getOperand(0));
1305 continue;
1306 }
1307 LoopWorklist.push_back(Op.getNode());
1308 ChainOps.push_back(Op);
1309 }
1310
1311 if (ChainCheck) {
1312 // Add the other operand of StoredVal to worklist.
1313 for (SDValue Op : StoredVal->ops())
1314 if (Op.getNode() != LoadNode)
1315 LoopWorklist.push_back(Op.getNode());
1316
1317 // Check if Load is reachable from any of the nodes in the worklist.
1318 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1319 true))
1320 return false;
1321
1322 // Make a new TokenFactor with all the other input chains except
1323 // for the load.
1324 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1325 MVT::Other, ChainOps);
1326 }
1327 }
1328 if (!ChainCheck)
1329 return false;
1330
1331 return true;
1332}
1333
1334// Change a chain of {load; op; store} of the same value into a simple op
1335// through memory of that value, if the uses of the modified value and its
1336// address are suitable.
1337//
1338// The tablegen pattern memory operand pattern is currently not able to match
1339// the case where the CC on the original operation are used.
1340//
1341// See the equivalent routine in X86ISelDAGToDAG for further comments.
1342bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1343 StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1344 SDValue StoredVal = StoreNode->getOperand(1);
1345 unsigned Opc = StoredVal->getOpcode();
1346 SDLoc DL(StoreNode);
1347
1348 // Before we try to select anything, make sure this is memory operand size
1349 // and opcode we can handle. Note that this must match the code below that
1350 // actually lowers the opcodes.
1351 EVT MemVT = StoreNode->getMemoryVT();
1352 unsigned NewOpc = 0;
1353 bool NegateOperand = false;
1354 switch (Opc) {
1355 default:
1356 return false;
1357 case SystemZISD::SSUBO:
1358 NegateOperand = true;
1359 [[fallthrough]];
1360 case SystemZISD::SADDO:
1361 if (MemVT == MVT::i32)
1362 NewOpc = SystemZ::ASI;
1363 else if (MemVT == MVT::i64)
1364 NewOpc = SystemZ::AGSI;
1365 else
1366 return false;
1367 break;
1368 case SystemZISD::USUBO:
1369 NegateOperand = true;
1370 [[fallthrough]];
1371 case SystemZISD::UADDO:
1372 if (MemVT == MVT::i32)
1373 NewOpc = SystemZ::ALSI;
1374 else if (MemVT == MVT::i64)
1375 NewOpc = SystemZ::ALGSI;
1376 else
1377 return false;
1378 break;
1379 }
1380
1381 LoadSDNode *LoadNode = nullptr;
1382 SDValue InputChain;
1383 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1384 InputChain))
1385 return false;
1386
1387 SDValue Operand = StoredVal.getOperand(1);
1388 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1389 if (!OperandC)
1390 return false;
1391 auto OperandV = OperandC->getAPIntValue();
1392 if (NegateOperand)
1393 OperandV = -OperandV;
1394 if (OperandV.getSignificantBits() > 8)
1395 return false;
1396 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1397
1398 SDValue Base, Disp;
1399 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1400 return false;
1401
1402 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1404 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1405 CurDAG->setNodeMemRefs(
1406 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1407
1408 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1409 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1410 CurDAG->RemoveDeadNode(Node);
1411 return true;
1412}
1413
1414bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1415 LoadSDNode *Load) const {
1416 // Check that the two memory operands have the same size.
1417 if (Load->getMemoryVT() != Store->getMemoryVT())
1418 return false;
1419
1420 // Volatility stops an access from being decomposed.
1421 if (Load->isVolatile() || Store->isVolatile())
1422 return false;
1423
1424 // There's no chance of overlap if the load is invariant.
1425 if (Load->isInvariant() && Load->isDereferenceable())
1426 return true;
1427
1428 // Otherwise we need to check whether there's an alias.
1429 const Value *V1 = Load->getMemOperand()->getValue();
1430 const Value *V2 = Store->getMemOperand()->getValue();
1431 if (!V1 || !V2)
1432 return false;
1433
1434 // Reject equality.
1435 uint64_t Size = Load->getMemoryVT().getStoreSize();
1436 int64_t End1 = Load->getSrcValueOffset() + Size;
1437 int64_t End2 = Store->getSrcValueOffset() + Size;
1438 if (V1 == V2 && End1 == End2)
1439 return false;
1440
1441 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
1442 MemoryLocation(V2, End2, Store->getAAInfo()));
1443}
1444
1445bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1446 auto *Store = cast<StoreSDNode>(N);
1447 auto *Load = cast<LoadSDNode>(Store->getValue());
1448
1449 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1450 // instructions.
1451 uint64_t Size = Load->getMemoryVT().getStoreSize();
1452 if (Size > 1 && Size <= 8) {
1453 // Prefer LHRL, LRL and LGRL.
1454 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1455 return false;
1456 // Prefer STHRL, STRL and STGRL.
1457 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1458 return false;
1459 }
1460
1461 return canUseBlockOperation(Store, Load);
1462}
1463
1464bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1465 unsigned I) const {
1466 auto *StoreA = cast<StoreSDNode>(N);
1467 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1468 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1469 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1470 canUseBlockOperation(StoreA, LoadB);
1471}
1472
1473bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1474
1475 auto *MemAccess = cast<LSBaseSDNode>(N);
1476 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1477 SDValue BasePtr = MemAccess->getBasePtr();
1478 MachineMemOperand *MMO = MemAccess->getMemOperand();
1479 assert(MMO && "Expected a memory operand.");
1480
1481 // The memory access must have a proper alignment and no index register.
1482 if (MemAccess->getAlign().value() < StoreSize ||
1483 !MemAccess->getOffset().isUndef())
1484 return false;
1485
1486 // The MMO must not have an unaligned offset.
1487 if (MMO->getOffset() % StoreSize != 0)
1488 return false;
1489
1490 // An access to GOT or the Constant Pool is aligned.
1491 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1492 if ((PSV->isGOT() || PSV->isConstantPool()))
1493 return true;
1494
1495 // Check the alignment of a Global Address.
1496 if (BasePtr.getNumOperands())
1497 if (GlobalAddressSDNode *GA =
1498 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) {
1499 // The immediate offset must be aligned.
1500 if (GA->getOffset() % StoreSize != 0)
1501 return false;
1502
1503 // The alignment of the symbol itself must be at least the store size.
1504 const GlobalValue *GV = GA->getGlobal();
1505 const DataLayout &DL = GV->getParent()->getDataLayout();
1506 if (GV->getPointerAlignment(DL).value() < StoreSize)
1507 return false;
1508 }
1509
1510 return true;
1511}
1512
1513void SystemZDAGToDAGISel::Select(SDNode *Node) {
1514 // If we have a custom node, we already have selected!
1515 if (Node->isMachineOpcode()) {
1516 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1517 Node->setNodeId(-1);
1518 return;
1519 }
1520
1521 unsigned Opcode = Node->getOpcode();
1522 switch (Opcode) {
1523 case ISD::OR:
1524 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1525 if (tryRxSBG(Node, SystemZ::ROSBG))
1526 return;
1527 goto or_xor;
1528
1529 case ISD::XOR:
1530 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1531 if (tryRxSBG(Node, SystemZ::RXSBG))
1532 return;
1533 // Fall through.
1534 or_xor:
1535 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1536 // split the operation into two. If both operands here happen to be
1537 // constant, leave this to common code to optimize.
1538 if (Node->getValueType(0) == MVT::i64 &&
1539 Node->getOperand(0).getOpcode() != ISD::Constant)
1540 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1541 uint64_t Val = Op1->getZExtValue();
1542 // Don't split the operation if we can match one of the combined
1543 // logical operations provided by miscellaneous-extensions-3.
1544 if (Subtarget->hasMiscellaneousExtensions3()) {
1545 unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1546 // Check whether this expression matches NAND/NOR/NXOR.
1547 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1548 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1549 ChildOpcode == ISD::XOR)
1550 break;
1551 // Check whether this expression matches OR-with-complement
1552 // (or matches an alternate pattern for NXOR).
1553 if (ChildOpcode == ISD::XOR) {
1554 auto Op0 = Node->getOperand(0);
1555 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1556 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1557 break;
1558 }
1559 }
1560 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1561 splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1562 Val - uint32_t(Val), uint32_t(Val));
1563 return;
1564 }
1565 }
1566 break;
1567
1568 case ISD::AND:
1569 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1570 if (tryRxSBG(Node, SystemZ::RNSBG))
1571 return;
1572 [[fallthrough]];
1573 case ISD::ROTL:
1574 case ISD::SHL:
1575 case ISD::SRL:
1576 case ISD::ZERO_EXTEND:
1577 if (tryRISBGZero(Node))
1578 return;
1579 break;
1580
1581 case ISD::Constant:
1582 // If this is a 64-bit constant that is out of the range of LLILF,
1583 // LLIHF and LGFI, split it into two 32-bit pieces.
1584 if (Node->getValueType(0) == MVT::i64) {
1585 uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1586 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1587 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1588 uint32_t(Val));
1589 return;
1590 }
1591 }
1592 break;
1593
1595 SDValue Op0 = Node->getOperand(0);
1596 SDValue Op1 = Node->getOperand(1);
1597 // Prefer to put any load first, so that it can be matched as a
1598 // conditional load. Likewise for constants in range for LOCHI.
1599 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1600 (Subtarget->hasLoadStoreOnCond2() &&
1601 Node->getValueType(0).isInteger() &&
1602 Op1.getOpcode() == ISD::Constant &&
1603 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1604 !(Op0.getOpcode() == ISD::Constant &&
1605 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1606 SDValue CCValid = Node->getOperand(2);
1607 SDValue CCMask = Node->getOperand(3);
1608 uint64_t ConstCCValid =
1609 cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1610 uint64_t ConstCCMask =
1611 cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1612 // Invert the condition.
1613 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1614 SDLoc(Node), CCMask.getValueType());
1615 SDValue Op4 = Node->getOperand(4);
1616 SDNode *UpdatedNode =
1617 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1618 if (UpdatedNode != Node) {
1619 // In case this node already exists then replace Node with it.
1620 ReplaceNode(Node, UpdatedNode);
1621 Node = UpdatedNode;
1622 }
1623 }
1624 break;
1625 }
1626
1628 EVT VT = Node->getValueType(0);
1629 unsigned ElemBitSize = VT.getScalarSizeInBits();
1630 if (ElemBitSize == 32) {
1631 if (tryGather(Node, SystemZ::VGEF))
1632 return;
1633 } else if (ElemBitSize == 64) {
1634 if (tryGather(Node, SystemZ::VGEG))
1635 return;
1636 }
1637 break;
1638 }
1639
1640 case ISD::BUILD_VECTOR: {
1641 auto *BVN = cast<BuildVectorSDNode>(Node);
1643 if (VCI.isVectorConstantLegal(*Subtarget)) {
1644 loadVectorConstant(VCI, Node);
1645 return;
1646 }
1647 break;
1648 }
1649
1650 case ISD::ConstantFP: {
1651 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1652 if (Imm.isZero() || Imm.isNegZero())
1653 break;
1655 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1656 assert(Success && "Expected legal FP immediate");
1657 loadVectorConstant(VCI, Node);
1658 return;
1659 }
1660
1661 case ISD::STORE: {
1662 if (tryFoldLoadStoreIntoMemOperand(Node))
1663 return;
1664 auto *Store = cast<StoreSDNode>(Node);
1665 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1666 if (ElemBitSize == 32) {
1667 if (tryScatter(Store, SystemZ::VSCEF))
1668 return;
1669 } else if (ElemBitSize == 64) {
1670 if (tryScatter(Store, SystemZ::VSCEG))
1671 return;
1672 }
1673 break;
1674 }
1675 }
1676
1677 SelectCode(Node);
1678}
1679
1680bool SystemZDAGToDAGISel::
1681SelectInlineAsmMemoryOperand(const SDValue &Op,
1682 unsigned ConstraintID,
1683 std::vector<SDValue> &OutOps) {
1684 SystemZAddressingMode::AddrForm Form;
1685 SystemZAddressingMode::DispRange DispRange;
1686 SDValue Base, Disp, Index;
1687
1688 switch(ConstraintID) {
1689 default:
1690 llvm_unreachable("Unexpected asm memory constraint");
1694 // Accept an address with a short displacement, but no index.
1695 Form = SystemZAddressingMode::FormBD;
1696 DispRange = SystemZAddressingMode::Disp12Only;
1697 break;
1700 // Accept an address with a short displacement and an index.
1701 Form = SystemZAddressingMode::FormBDXNormal;
1702 DispRange = SystemZAddressingMode::Disp12Only;
1703 break;
1706 // Accept an address with a long displacement, but no index.
1707 Form = SystemZAddressingMode::FormBD;
1708 DispRange = SystemZAddressingMode::Disp20Only;
1709 break;
1715 // Accept an address with a long displacement and an index.
1716 // m works the same as T, as this is the most general case.
1717 // We don't really have any special handling of "offsettable"
1718 // memory addresses, so just treat o the same as m.
1719 Form = SystemZAddressingMode::FormBDXNormal;
1720 DispRange = SystemZAddressingMode::Disp20Only;
1721 break;
1722 }
1723
1724 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1725 const TargetRegisterClass *TRC =
1726 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1727 SDLoc DL(Base);
1728 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1729
1730 // Make sure that the base address doesn't go into %r0.
1731 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1732 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1733 Base.getOpcode() != ISD::Register) {
1734 Base =
1735 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1736 DL, Base.getValueType(),
1737 Base, RC), 0);
1738 }
1739
1740 // Make sure that the index register isn't assigned to %r0 either.
1741 if (Index.getOpcode() != ISD::Register) {
1742 Index =
1743 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1744 DL, Index.getValueType(),
1745 Index, RC), 0);
1746 }
1747
1748 OutOps.push_back(Base);
1749 OutOps.push_back(Disp);
1750 OutOps.push_back(Index);
1751 return false;
1752 }
1753
1754 return true;
1755}
1756
1757// IsProfitableToFold - Returns true if is profitable to fold the specific
1758// operand node N of U during instruction selection that starts at Root.
1759bool
1760SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1761 SDNode *Root) const {
1762 // We want to avoid folding a LOAD into an ICMP node if as a result
1763 // we would be forced to spill the condition code into a GPR.
1764 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1765 if (!N.hasOneUse() || !U->hasOneUse())
1766 return false;
1767
1768 // The user of the CC value will usually be a CopyToReg into the
1769 // physical CC register, which in turn is glued and chained to the
1770 // actual instruction that uses the CC value. Bail out if we have
1771 // anything else than that.
1772 SDNode *CCUser = *U->use_begin();
1773 SDNode *CCRegUser = nullptr;
1774 if (CCUser->getOpcode() == ISD::CopyToReg ||
1775 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1776 for (auto *U : CCUser->uses()) {
1777 if (CCRegUser == nullptr)
1778 CCRegUser = U;
1779 else if (CCRegUser != U)
1780 return false;
1781 }
1782 }
1783 if (CCRegUser == nullptr)
1784 return false;
1785
1786 // If the actual instruction is a branch, the only thing that remains to be
1787 // checked is whether the CCUser chain is a predecessor of the load.
1788 if (CCRegUser->isMachineOpcode() &&
1789 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1790 return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1791
1792 // Otherwise, the instruction may have multiple operands, and we need to
1793 // verify that none of them are a predecessor of the load. This is exactly
1794 // the same check that would be done by common code if the CC setter were
1795 // glued to the CC user, so simply invoke that check here.
1796 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1797 return false;
1798 }
1799
1800 return true;
1801}
1802
1803namespace {
1804// Represents a sequence for extracting a 0/1 value from an IPM result:
1805// (((X ^ XORValue) + AddValue) >> Bit)
1806struct IPMConversion {
1807 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1808 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1809
1810 int64_t XORValue;
1811 int64_t AddValue;
1812 unsigned Bit;
1813};
1814} // end anonymous namespace
1815
1816// Return a sequence for getting a 1 from an IPM result when CC has a
1817// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1818// The handling of CC values outside CCValid doesn't matter.
1819static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1820 // Deal with cases where the result can be taken directly from a bit
1821 // of the IPM result.
1822 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1823 return IPMConversion(0, 0, SystemZ::IPM_CC);
1824 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1825 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1826
1827 // Deal with cases where we can add a value to force the sign bit
1828 // to contain the right value. Putting the bit in 31 means we can
1829 // use SRL rather than RISBG(L), and also makes it easier to get a
1830 // 0/-1 value, so it has priority over the other tests below.
1831 //
1832 // These sequences rely on the fact that the upper two bits of the
1833 // IPM result are zero.
1834 uint64_t TopBit = uint64_t(1) << 31;
1835 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1836 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1837 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1838 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1839 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1842 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1843 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1844 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1845 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1848 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1849
1850 // Next try inverting the value and testing a bit. 0/1 could be
1851 // handled this way too, but we dealt with that case above.
1852 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1853 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1854
1855 // Handle cases where adding a value forces a non-sign bit to contain
1856 // the right value.
1857 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1858 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1859 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1860 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1861
1862 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1863 // can be done by inverting the low CC bit and applying one of the
1864 // sign-based extractions above.
1865 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1866 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1867 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1868 return IPMConversion(1 << SystemZ::IPM_CC,
1869 TopBit - (3 << SystemZ::IPM_CC), 31);
1870 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1873 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1874 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1877 return IPMConversion(1 << SystemZ::IPM_CC,
1878 TopBit - (1 << SystemZ::IPM_CC), 31);
1879
1880 llvm_unreachable("Unexpected CC combination");
1881}
1882
1883SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1884 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1885 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1886 if (!TrueOp || !FalseOp)
1887 return SDValue();
1888 if (FalseOp->getZExtValue() != 0)
1889 return SDValue();
1890 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1891 return SDValue();
1892
1893 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1894 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1895 if (!CCValidOp || !CCMaskOp)
1896 return SDValue();
1897 int CCValid = CCValidOp->getZExtValue();
1898 int CCMask = CCMaskOp->getZExtValue();
1899
1900 SDLoc DL(Node);
1901 SDValue CCReg = Node->getOperand(4);
1902 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1903 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1904
1905 if (IPM.XORValue)
1906 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1907 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1908
1909 if (IPM.AddValue)
1910 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1911 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1912
1913 EVT VT = Node->getValueType(0);
1914 if (VT == MVT::i32 && IPM.Bit == 31) {
1915 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
1916 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1917 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1918 } else {
1919 if (VT != MVT::i32)
1920 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1921
1922 if (TrueOp->getSExtValue() == 1) {
1923 // The SHR/AND sequence should get optimized to an RISBG.
1924 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1925 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1926 Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1927 CurDAG->getConstant(1, DL, VT));
1928 } else {
1929 // Sign-extend from IPM.Bit using a pair of shifts.
1930 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
1931 int SraAmt = VT.getSizeInBits() - 1;
1932 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
1933 CurDAG->getConstant(ShlAmt, DL, MVT::i32));
1934 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
1935 CurDAG->getConstant(SraAmt, DL, MVT::i32));
1936 }
1937 }
1938
1939 return Result;
1940}
1941
1942void SystemZDAGToDAGISel::PreprocessISelDAG() {
1943 // If we have conditional immediate loads, we always prefer
1944 // using those over an IPM sequence.
1945 if (Subtarget->hasLoadStoreOnCond2())
1946 return;
1947
1948 bool MadeChange = false;
1949
1950 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
1951 E = CurDAG->allnodes_end();
1952 I != E;) {
1953 SDNode *N = &*I++;
1954 if (N->use_empty())
1955 continue;
1956
1957 SDValue Res;
1958 switch (N->getOpcode()) {
1959 default: break;
1961 Res = expandSelectBoolean(N);
1962 break;
1963 }
1964
1965 if (Res) {
1966 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
1967 LLVM_DEBUG(N->dump(CurDAG));
1968 LLVM_DEBUG(dbgs() << "\nNew: ");
1969 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
1970 LLVM_DEBUG(dbgs() << "\n");
1971
1972 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1973 MadeChange = true;
1974 }
1975 }
1976
1977 if (MadeChange)
1978 CurDAG->RemoveDeadNodes();
1979}
#define Success
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
bool End
Definition: ELF_riscv.cpp:464
const HexagonInstrInfo * TII
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
#define PASS_NAME
#define DEBUG_TYPE
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
DEMANGLE_DUMP_METHOD void dump() const
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1498
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
LLVM Value Representation.
Definition: Value.h:74
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition: Value.cpp:921
Iterator for intrusive lists based on ilist_node.
self_iterator getIterator()
Definition: ilist_node.h:82
#define UINT64_MAX
Definition: DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Level
Code generation optimization level.
Definition: CodeGen.h:57
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:978
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:779
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
@ FrameIndex
Definition: ISDOpcodes.h:80
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:704
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
@ TargetFrameIndex
Definition: ISDOpcodes.h:166
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:794
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:679
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:523
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
const unsigned CCMASK_0
Definition: SystemZ.h:27
const unsigned CCMASK_1
Definition: SystemZ.h:28
const unsigned IPM_CC
Definition: SystemZ.h:112
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:186
const unsigned CCMASK_3
Definition: SystemZ.h:30
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:181
const unsigned CCMASK_2
Definition: SystemZ.h:29
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition: DWP.cpp:406
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
#define N
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:363
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)