LLVM  6.0.0svn
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines an instruction selector for the SystemZ target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SystemZTargetMachine.h"
17 #include "llvm/Support/Debug.h"
18 #include "llvm/Support/KnownBits.h"
20 
21 using namespace llvm;
22 
23 #define DEBUG_TYPE "systemz-isel"
24 
25 namespace {
26 // Used to build addressing modes.
27 struct SystemZAddressingMode {
28  // The shape of the address.
29  enum AddrForm {
30  // base+displacement
31  FormBD,
32 
33  // base+displacement+index for load and store operands
34  FormBDXNormal,
35 
36  // base+displacement+index for load address operands
37  FormBDXLA,
38 
39  // base+displacement+index+ADJDYNALLOC
40  FormBDXDynAlloc
41  };
42  AddrForm Form;
43 
44  // The type of displacement. The enum names here correspond directly
45  // to the definitions in SystemZOperand.td. We could split them into
46  // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
47  enum DispRange {
48  Disp12Only,
49  Disp12Pair,
50  Disp20Only,
51  Disp20Only128,
52  Disp20Pair
53  };
54  DispRange DR;
55 
56  // The parts of the address. The address is equivalent to:
57  //
58  // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
59  SDValue Base;
60  int64_t Disp;
61  SDValue Index;
62  bool IncludesDynAlloc;
63 
64  SystemZAddressingMode(AddrForm form, DispRange dr)
65  : Form(form), DR(dr), Base(), Disp(0), Index(),
66  IncludesDynAlloc(false) {}
67 
68  // True if the address can have an index register.
69  bool hasIndexField() { return Form != FormBD; }
70 
71  // True if the address can (and must) include ADJDYNALLOC.
72  bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73 
74  void dump() {
75  errs() << "SystemZAddressingMode " << this << '\n';
76 
77  errs() << " Base ";
78  if (Base.getNode())
79  Base.getNode()->dump();
80  else
81  errs() << "null\n";
82 
83  if (hasIndexField()) {
84  errs() << " Index ";
85  if (Index.getNode())
86  Index.getNode()->dump();
87  else
88  errs() << "null\n";
89  }
90 
91  errs() << " Disp " << Disp;
92  if (IncludesDynAlloc)
93  errs() << " + ADJDYNALLOC";
94  errs() << '\n';
95  }
96 };
97 
98 // Return a mask with Count low bits set.
99 static uint64_t allOnes(unsigned int Count) {
100  assert(Count <= 64);
101  if (Count > 63)
102  return UINT64_MAX;
103  return (uint64_t(1) << Count) - 1;
104 }
105 
106 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108 // Rotate (I5). The combined operand value is effectively:
109 //
110 // (or (rotl Input, Rotate), ~Mask)
111 //
112 // for RNSBG and:
113 //
114 // (and (rotl Input, Rotate), Mask)
115 //
116 // otherwise. The output value has BitSize bits, although Input may be
117 // narrower (in which case the upper bits are don't care), or wider (in which
118 // case the result will be truncated as part of the operation).
119 struct RxSBGOperands {
120  RxSBGOperands(unsigned Op, SDValue N)
121  : Opcode(Op), BitSize(N.getValueSizeInBits()),
122  Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123  Rotate(0) {}
124 
125  unsigned Opcode;
126  unsigned BitSize;
127  uint64_t Mask;
128  SDValue Input;
129  unsigned Start;
130  unsigned End;
131  unsigned Rotate;
132 };
133 
134 class SystemZDAGToDAGISel : public SelectionDAGISel {
135  const SystemZSubtarget *Subtarget;
136 
137  // Used by SystemZOperands.td to create integer constants.
138  inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139  return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140  }
141 
142  const SystemZTargetMachine &getTargetMachine() const {
143  return static_cast<const SystemZTargetMachine &>(TM);
144  }
145 
146  const SystemZInstrInfo *getInstrInfo() const {
147  return Subtarget->getInstrInfo();
148  }
149 
150  // Try to fold more of the base or index of AM into AM, where IsBase
151  // selects between the base and index.
152  bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153 
154  // Try to describe N in AM, returning true on success.
155  bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156 
157  // Extract individual target operands from matched address AM.
158  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159  SDValue &Base, SDValue &Disp) const;
160  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161  SDValue &Base, SDValue &Disp, SDValue &Index) const;
162 
163  // Try to match Addr as a FormBD address with displacement type DR.
164  // Return true on success, storing the base and displacement in
165  // Base and Disp respectively.
166  bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167  SDValue &Base, SDValue &Disp) const;
168 
169  // Try to match Addr as a FormBDX address with displacement type DR.
170  // Return true on success and if the result had no index. Store the
171  // base and displacement in Base and Disp respectively.
172  bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173  SDValue &Base, SDValue &Disp) const;
174 
175  // Try to match Addr as a FormBDX* address of form Form with
176  // displacement type DR. Return true on success, storing the base,
177  // displacement and index in Base, Disp and Index respectively.
178  bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179  SystemZAddressingMode::DispRange DR, SDValue Addr,
180  SDValue &Base, SDValue &Disp, SDValue &Index) const;
181 
182  // PC-relative address matching routines used by SystemZOperands.td.
183  bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184  if (SystemZISD::isPCREL(Addr.getOpcode())) {
185  Target = Addr.getOperand(0);
186  return true;
187  }
188  return false;
189  }
190 
191  // BD matching routines used by SystemZOperands.td.
192  bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193  return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194  }
195  bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196  return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197  }
198  bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199  return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200  }
201  bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202  return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203  }
204 
205  // MVI matching routines used by SystemZOperands.td.
206  bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207  return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208  }
209  bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210  return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211  }
212 
213  // BDX matching routines used by SystemZOperands.td.
214  bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215  SDValue &Index) const {
216  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217  SystemZAddressingMode::Disp12Only,
218  Addr, Base, Disp, Index);
219  }
220  bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221  SDValue &Index) const {
222  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223  SystemZAddressingMode::Disp12Pair,
224  Addr, Base, Disp, Index);
225  }
226  bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227  SDValue &Index) const {
228  return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229  SystemZAddressingMode::Disp12Only,
230  Addr, Base, Disp, Index);
231  }
232  bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233  SDValue &Index) const {
234  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235  SystemZAddressingMode::Disp20Only,
236  Addr, Base, Disp, Index);
237  }
238  bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239  SDValue &Index) const {
240  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241  SystemZAddressingMode::Disp20Only128,
242  Addr, Base, Disp, Index);
243  }
244  bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245  SDValue &Index) const {
246  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247  SystemZAddressingMode::Disp20Pair,
248  Addr, Base, Disp, Index);
249  }
250  bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251  SDValue &Index) const {
252  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253  SystemZAddressingMode::Disp12Pair,
254  Addr, Base, Disp, Index);
255  }
256  bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257  SDValue &Index) const {
258  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259  SystemZAddressingMode::Disp20Pair,
260  Addr, Base, Disp, Index);
261  }
262 
263  // Try to match Addr as an address with a base, 12-bit displacement
264  // and index, where the index is element Elem of a vector.
265  // Return true on success, storing the base, displacement and vector
266  // in Base, Disp and Index respectively.
267  bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268  SDValue &Disp, SDValue &Index) const;
269 
270  // Check whether (or Op (and X InsertMask)) is effectively an insertion
271  // of X into bits InsertMask of some Y != Op. Return true if so and
272  // set Op to that Y.
273  bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274 
275  // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276  // Return true on success.
277  bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278 
279  // Try to fold some of RxSBG.Input into other fields of RxSBG.
280  // Return true on success.
281  bool expandRxSBG(RxSBGOperands &RxSBG) const;
282 
283  // Return an undefined value of type VT.
284  SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285 
286  // Convert N to VT, if it isn't already.
287  SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288 
289  // Try to implement AND or shift node N using RISBG with the zero flag set.
290  // Return the selected node on success, otherwise return null.
291  bool tryRISBGZero(SDNode *N);
292 
293  // Try to use RISBG or Opcode to implement OR or XOR node N.
294  // Return the selected node on success, otherwise return null.
295  bool tryRxSBG(SDNode *N, unsigned Opcode);
296 
297  // If Op0 is null, then Node is a constant that can be loaded using:
298  //
299  // (Opcode UpperVal LowerVal)
300  //
301  // If Op0 is nonnull, then Node can be implemented using:
302  //
303  // (Opcode (Opcode Op0 UpperVal) LowerVal)
304  void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305  uint64_t UpperVal, uint64_t LowerVal);
306 
307  // Try to use gather instruction Opcode to implement vector insertion N.
308  bool tryGather(SDNode *N, unsigned Opcode);
309 
310  // Try to use scatter instruction Opcode to implement store Store.
311  bool tryScatter(StoreSDNode *Store, unsigned Opcode);
312 
313  // Return true if Load and Store are loads and stores of the same size
314  // and are guaranteed not to overlap. Such operations can be implemented
315  // using block (SS-format) instructions.
316  //
317  // Partial overlap would lead to incorrect code, since the block operations
318  // are logically bytewise, even though they have a fast path for the
319  // non-overlapping case. We also need to avoid full overlap (i.e. two
320  // addresses that might be equal at run time) because although that case
321  // would be handled correctly, it might be implemented by millicode.
322  bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
323 
324  // N is a (store (load Y), X) pattern. Return true if it can use an MVC
325  // from Y to X.
326  bool storeLoadCanUseMVC(SDNode *N) const;
327 
328  // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
329  // if A[1 - I] == X and if N can use a block operation like NC from A[I]
330  // to X.
331  bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
332 
333 public:
334  SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
335  : SelectionDAGISel(TM, OptLevel) {}
336 
337  bool runOnMachineFunction(MachineFunction &MF) override {
338  Subtarget = &MF.getSubtarget<SystemZSubtarget>();
340  }
341 
342  // Override MachineFunctionPass.
343  StringRef getPassName() const override {
344  return "SystemZ DAG->DAG Pattern Instruction Selection";
345  }
346 
347  // Override SelectionDAGISel.
348  void Select(SDNode *Node) override;
349  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
350  std::vector<SDValue> &OutOps) override;
351 
352  // Include the pieces autogenerated from the target description.
353  #include "SystemZGenDAGISel.inc"
354 };
355 } // end anonymous namespace
356 
358  CodeGenOpt::Level OptLevel) {
359  return new SystemZDAGToDAGISel(TM, OptLevel);
360 }
361 
362 // Return true if Val should be selected as a displacement for an address
363 // with range DR. Here we're interested in the range of both the instruction
364 // described by DR and of any pairing instruction.
365 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
366  switch (DR) {
367  case SystemZAddressingMode::Disp12Only:
368  return isUInt<12>(Val);
369 
370  case SystemZAddressingMode::Disp12Pair:
371  case SystemZAddressingMode::Disp20Only:
372  case SystemZAddressingMode::Disp20Pair:
373  return isInt<20>(Val);
374 
375  case SystemZAddressingMode::Disp20Only128:
376  return isInt<20>(Val) && isInt<20>(Val + 8);
377  }
378  llvm_unreachable("Unhandled displacement range");
379 }
380 
381 // Change the base or index in AM to Value, where IsBase selects
382 // between the base and index.
383 static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
384  SDValue Value) {
385  if (IsBase)
386  AM.Base = Value;
387  else
388  AM.Index = Value;
389 }
390 
391 // The base or index of AM is equivalent to Value + ADJDYNALLOC,
392 // where IsBase selects between the base and index. Try to fold the
393 // ADJDYNALLOC into AM.
394 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
395  SDValue Value) {
396  if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
397  changeComponent(AM, IsBase, Value);
398  AM.IncludesDynAlloc = true;
399  return true;
400  }
401  return false;
402 }
403 
404 // The base of AM is equivalent to Base + Index. Try to use Index as
405 // the index register.
406 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
407  SDValue Index) {
408  if (AM.hasIndexField() && !AM.Index.getNode()) {
409  AM.Base = Base;
410  AM.Index = Index;
411  return true;
412  }
413  return false;
414 }
415 
416 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
417 // between the base and index. Try to fold Op1 into AM's displacement.
418 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
419  SDValue Op0, uint64_t Op1) {
420  // First try adjusting the displacement.
421  int64_t TestDisp = AM.Disp + Op1;
422  if (selectDisp(AM.DR, TestDisp)) {
423  changeComponent(AM, IsBase, Op0);
424  AM.Disp = TestDisp;
425  return true;
426  }
427 
428  // We could consider forcing the displacement into a register and
429  // using it as an index, but it would need to be carefully tuned.
430  return false;
431 }
432 
433 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
434  bool IsBase) const {
435  SDValue N = IsBase ? AM.Base : AM.Index;
436  unsigned Opcode = N.getOpcode();
437  if (Opcode == ISD::TRUNCATE) {
438  N = N.getOperand(0);
439  Opcode = N.getOpcode();
440  }
441  if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
442  SDValue Op0 = N.getOperand(0);
443  SDValue Op1 = N.getOperand(1);
444 
445  unsigned Op0Code = Op0->getOpcode();
446  unsigned Op1Code = Op1->getOpcode();
447 
448  if (Op0Code == SystemZISD::ADJDYNALLOC)
449  return expandAdjDynAlloc(AM, IsBase, Op1);
450  if (Op1Code == SystemZISD::ADJDYNALLOC)
451  return expandAdjDynAlloc(AM, IsBase, Op0);
452 
453  if (Op0Code == ISD::Constant)
454  return expandDisp(AM, IsBase, Op1,
455  cast<ConstantSDNode>(Op0)->getSExtValue());
456  if (Op1Code == ISD::Constant)
457  return expandDisp(AM, IsBase, Op0,
458  cast<ConstantSDNode>(Op1)->getSExtValue());
459 
460  if (IsBase && expandIndex(AM, Op0, Op1))
461  return true;
462  }
463  if (Opcode == SystemZISD::PCREL_OFFSET) {
464  SDValue Full = N.getOperand(0);
465  SDValue Base = N.getOperand(1);
466  SDValue Anchor = Base.getOperand(0);
467  uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
468  cast<GlobalAddressSDNode>(Anchor)->getOffset());
469  return expandDisp(AM, IsBase, Base, Offset);
470  }
471  return false;
472 }
473 
474 // Return true if an instruction with displacement range DR should be
475 // used for displacement value Val. selectDisp(DR, Val) must already hold.
476 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
477  assert(selectDisp(DR, Val) && "Invalid displacement");
478  switch (DR) {
479  case SystemZAddressingMode::Disp12Only:
480  case SystemZAddressingMode::Disp20Only:
481  case SystemZAddressingMode::Disp20Only128:
482  return true;
483 
484  case SystemZAddressingMode::Disp12Pair:
485  // Use the other instruction if the displacement is too large.
486  return isUInt<12>(Val);
487 
488  case SystemZAddressingMode::Disp20Pair:
489  // Use the other instruction if the displacement is small enough.
490  return !isUInt<12>(Val);
491  }
492  llvm_unreachable("Unhandled displacement range");
493 }
494 
495 // Return true if Base + Disp + Index should be performed by LA(Y).
496 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
497  // Don't use LA(Y) for constants.
498  if (!Base)
499  return false;
500 
501  // Always use LA(Y) for frame addresses, since we know that the destination
502  // register is almost always (perhaps always) going to be different from
503  // the frame register.
504  if (Base->getOpcode() == ISD::FrameIndex)
505  return true;
506 
507  if (Disp) {
508  // Always use LA(Y) if there is a base, displacement and index.
509  if (Index)
510  return true;
511 
512  // Always use LA if the displacement is small enough. It should always
513  // be no worse than AGHI (and better if it avoids a move).
514  if (isUInt<12>(Disp))
515  return true;
516 
517  // For similar reasons, always use LAY if the constant is too big for AGHI.
518  // LAY should be no worse than AGFI.
519  if (!isInt<16>(Disp))
520  return true;
521  } else {
522  // Don't use LA for plain registers.
523  if (!Index)
524  return false;
525 
526  // Don't use LA for plain addition if the index operand is only used
527  // once. It should be a natural two-operand addition in that case.
528  if (Index->hasOneUse())
529  return false;
530 
531  // Prefer addition if the second operation is sign-extended, in the
532  // hope of using AGF.
533  unsigned IndexOpcode = Index->getOpcode();
534  if (IndexOpcode == ISD::SIGN_EXTEND ||
535  IndexOpcode == ISD::SIGN_EXTEND_INREG)
536  return false;
537  }
538 
539  // Don't use LA for two-operand addition if either operand is only
540  // used once. The addition instructions are better in that case.
541  if (Base->hasOneUse())
542  return false;
543 
544  return true;
545 }
546 
547 // Return true if Addr is suitable for AM, updating AM if so.
548 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
549  SystemZAddressingMode &AM) const {
550  // Start out assuming that the address will need to be loaded separately,
551  // then try to extend it as much as we can.
552  AM.Base = Addr;
553 
554  // First try treating the address as a constant.
555  if (Addr.getOpcode() == ISD::Constant &&
556  expandDisp(AM, true, SDValue(),
557  cast<ConstantSDNode>(Addr)->getSExtValue()))
558  ;
559  // Also see if it's a bare ADJDYNALLOC.
560  else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
561  expandAdjDynAlloc(AM, true, SDValue()))
562  ;
563  else
564  // Otherwise try expanding each component.
565  while (expandAddress(AM, true) ||
566  (AM.Index.getNode() && expandAddress(AM, false)))
567  continue;
568 
569  // Reject cases where it isn't profitable to use LA(Y).
570  if (AM.Form == SystemZAddressingMode::FormBDXLA &&
571  !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
572  return false;
573 
574  // Reject cases where the other instruction in a pair should be used.
575  if (!isValidDisp(AM.DR, AM.Disp))
576  return false;
577 
578  // Make sure that ADJDYNALLOC is included where necessary.
579  if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
580  return false;
581 
582  DEBUG(AM.dump());
583  return true;
584 }
585 
586 // Insert a node into the DAG at least before Pos. This will reposition
587 // the node as needed, and will assign it a node ID that is <= Pos's ID.
588 // Note that this does *not* preserve the uniqueness of node IDs!
589 // The selection DAG must no longer depend on their uniqueness when this
590 // function is used.
591 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
592  if (N.getNode()->getNodeId() == -1 ||
593  N.getNode()->getNodeId() > Pos->getNodeId()) {
594  DAG->RepositionNode(Pos->getIterator(), N.getNode());
595  N.getNode()->setNodeId(Pos->getNodeId());
596  }
597 }
598 
599 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
600  EVT VT, SDValue &Base,
601  SDValue &Disp) const {
602  Base = AM.Base;
603  if (!Base.getNode())
604  // Register 0 means "no base". This is mostly useful for shifts.
605  Base = CurDAG->getRegister(0, VT);
606  else if (Base.getOpcode() == ISD::FrameIndex) {
607  // Lower a FrameIndex to a TargetFrameIndex.
608  int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
609  Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
610  } else if (Base.getValueType() != VT) {
611  // Truncate values from i64 to i32, for shifts.
612  assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
613  "Unexpected truncation");
614  SDLoc DL(Base);
615  SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
616  insertDAGNode(CurDAG, Base.getNode(), Trunc);
617  Base = Trunc;
618  }
619 
620  // Lower the displacement to a TargetConstant.
621  Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
622 }
623 
624 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
625  EVT VT, SDValue &Base,
626  SDValue &Disp,
627  SDValue &Index) const {
628  getAddressOperands(AM, VT, Base, Disp);
629 
630  Index = AM.Index;
631  if (!Index.getNode())
632  // Register 0 means "no index".
633  Index = CurDAG->getRegister(0, VT);
634 }
635 
636 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
637  SDValue Addr, SDValue &Base,
638  SDValue &Disp) const {
639  SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
640  if (!selectAddress(Addr, AM))
641  return false;
642 
643  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
644  return true;
645 }
646 
647 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
648  SDValue Addr, SDValue &Base,
649  SDValue &Disp) const {
650  SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
651  if (!selectAddress(Addr, AM) || AM.Index.getNode())
652  return false;
653 
654  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
655  return true;
656 }
657 
658 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
659  SystemZAddressingMode::DispRange DR,
660  SDValue Addr, SDValue &Base,
661  SDValue &Disp, SDValue &Index) const {
662  SystemZAddressingMode AM(Form, DR);
663  if (!selectAddress(Addr, AM))
664  return false;
665 
666  getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
667  return true;
668 }
669 
670 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
671  SDValue &Base,
672  SDValue &Disp,
673  SDValue &Index) const {
674  SDValue Regs[2];
675  if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
676  Regs[0].getNode() && Regs[1].getNode()) {
677  for (unsigned int I = 0; I < 2; ++I) {
678  Base = Regs[I];
679  Index = Regs[1 - I];
680  // We can't tell here whether the index vector has the right type
681  // for the access; the caller needs to do that instead.
682  if (Index.getOpcode() == ISD::ZERO_EXTEND)
683  Index = Index.getOperand(0);
684  if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
685  Index.getOperand(1) == Elem) {
686  Index = Index.getOperand(0);
687  return true;
688  }
689  }
690  }
691  return false;
692 }
693 
694 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
695  uint64_t InsertMask) const {
696  // We're only interested in cases where the insertion is into some operand
697  // of Op, rather than into Op itself. The only useful case is an AND.
698  if (Op.getOpcode() != ISD::AND)
699  return false;
700 
701  // We need a constant mask.
702  auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
703  if (!MaskNode)
704  return false;
705 
706  // It's not an insertion of Op.getOperand(0) if the two masks overlap.
707  uint64_t AndMask = MaskNode->getZExtValue();
708  if (InsertMask & AndMask)
709  return false;
710 
711  // It's only an insertion if all bits are covered or are known to be zero.
712  // The inner check covers all cases but is more expensive.
713  uint64_t Used = allOnes(Op.getValueSizeInBits());
714  if (Used != (AndMask | InsertMask)) {
715  KnownBits Known;
716  CurDAG->computeKnownBits(Op.getOperand(0), Known);
717  if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
718  return false;
719  }
720 
721  Op = Op.getOperand(0);
722  return true;
723 }
724 
725 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
726  uint64_t Mask) const {
727  const SystemZInstrInfo *TII = getInstrInfo();
728  if (RxSBG.Rotate != 0)
729  Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
730  Mask &= RxSBG.Mask;
731  if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
732  RxSBG.Mask = Mask;
733  return true;
734  }
735  return false;
736 }
737 
738 // Return true if any bits of (RxSBG.Input & Mask) are significant.
739 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
740  // Rotate the mask in the same way as RxSBG.Input is rotated.
741  if (RxSBG.Rotate != 0)
742  Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
743  return (Mask & RxSBG.Mask) != 0;
744 }
745 
746 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
747  SDValue N = RxSBG.Input;
748  unsigned Opcode = N.getOpcode();
749  switch (Opcode) {
750  case ISD::TRUNCATE: {
751  if (RxSBG.Opcode == SystemZ::RNSBG)
752  return false;
753  uint64_t BitSize = N.getValueSizeInBits();
754  uint64_t Mask = allOnes(BitSize);
755  if (!refineRxSBGMask(RxSBG, Mask))
756  return false;
757  RxSBG.Input = N.getOperand(0);
758  return true;
759  }
760  case ISD::AND: {
761  if (RxSBG.Opcode == SystemZ::RNSBG)
762  return false;
763 
764  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
765  if (!MaskNode)
766  return false;
767 
768  SDValue Input = N.getOperand(0);
769  uint64_t Mask = MaskNode->getZExtValue();
770  if (!refineRxSBGMask(RxSBG, Mask)) {
771  // If some bits of Input are already known zeros, those bits will have
772  // been removed from the mask. See if adding them back in makes the
773  // mask suitable.
774  KnownBits Known;
775  CurDAG->computeKnownBits(Input, Known);
776  Mask |= Known.Zero.getZExtValue();
777  if (!refineRxSBGMask(RxSBG, Mask))
778  return false;
779  }
780  RxSBG.Input = Input;
781  return true;
782  }
783 
784  case ISD::OR: {
785  if (RxSBG.Opcode != SystemZ::RNSBG)
786  return false;
787 
788  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
789  if (!MaskNode)
790  return false;
791 
792  SDValue Input = N.getOperand(0);
793  uint64_t Mask = ~MaskNode->getZExtValue();
794  if (!refineRxSBGMask(RxSBG, Mask)) {
795  // If some bits of Input are already known ones, those bits will have
796  // been removed from the mask. See if adding them back in makes the
797  // mask suitable.
798  KnownBits Known;
799  CurDAG->computeKnownBits(Input, Known);
800  Mask &= ~Known.One.getZExtValue();
801  if (!refineRxSBGMask(RxSBG, Mask))
802  return false;
803  }
804  RxSBG.Input = Input;
805  return true;
806  }
807 
808  case ISD::ROTL: {
809  // Any 64-bit rotate left can be merged into the RxSBG.
810  if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
811  return false;
812  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
813  if (!CountNode)
814  return false;
815 
816  RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
817  RxSBG.Input = N.getOperand(0);
818  return true;
819  }
820 
821  case ISD::ANY_EXTEND:
822  // Bits above the extended operand are don't-care.
823  RxSBG.Input = N.getOperand(0);
824  return true;
825 
826  case ISD::ZERO_EXTEND:
827  if (RxSBG.Opcode != SystemZ::RNSBG) {
828  // Restrict the mask to the extended operand.
829  unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
830  if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
831  return false;
832 
833  RxSBG.Input = N.getOperand(0);
834  return true;
835  }
837 
838  case ISD::SIGN_EXTEND: {
839  // Check that the extension bits are don't-care (i.e. are masked out
840  // by the final mask).
841  unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
842  if (maskMatters(RxSBG, allOnes(RxSBG.BitSize) - allOnes(InnerBitSize)))
843  return false;
844 
845  RxSBG.Input = N.getOperand(0);
846  return true;
847  }
848 
849  case ISD::SHL: {
850  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
851  if (!CountNode)
852  return false;
853 
854  uint64_t Count = CountNode->getZExtValue();
855  unsigned BitSize = N.getValueSizeInBits();
856  if (Count < 1 || Count >= BitSize)
857  return false;
858 
859  if (RxSBG.Opcode == SystemZ::RNSBG) {
860  // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
861  // count bits from RxSBG.Input are ignored.
862  if (maskMatters(RxSBG, allOnes(Count)))
863  return false;
864  } else {
865  // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
866  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
867  return false;
868  }
869 
870  RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
871  RxSBG.Input = N.getOperand(0);
872  return true;
873  }
874 
875  case ISD::SRL:
876  case ISD::SRA: {
877  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
878  if (!CountNode)
879  return false;
880 
881  uint64_t Count = CountNode->getZExtValue();
882  unsigned BitSize = N.getValueSizeInBits();
883  if (Count < 1 || Count >= BitSize)
884  return false;
885 
886  if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
887  // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
888  // count bits from RxSBG.Input are ignored.
889  if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
890  return false;
891  } else {
892  // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
893  // which is similar to SLL above.
894  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
895  return false;
896  }
897 
898  RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
899  RxSBG.Input = N.getOperand(0);
900  return true;
901  }
902  default:
903  return false;
904  }
905 }
906 
907 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
908  SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
909  return SDValue(N, 0);
910 }
911 
912 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
913  SDValue N) const {
914  if (N.getValueType() == MVT::i32 && VT == MVT::i64)
915  return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
916  DL, VT, getUNDEF(DL, MVT::i64), N);
917  if (N.getValueType() == MVT::i64 && VT == MVT::i32)
918  return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
919  assert(N.getValueType() == VT && "Unexpected value types");
920  return N;
921 }
922 
923 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
924  SDLoc DL(N);
925  EVT VT = N->getValueType(0);
926  if (!VT.isInteger() || VT.getSizeInBits() > 64)
927  return false;
928  RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
929  unsigned Count = 0;
930  while (expandRxSBG(RISBG))
931  // The widening or narrowing is expected to be free.
932  // Counting widening or narrowing as a saved operation will result in
933  // preferring an R*SBG over a simple shift/logical instruction.
934  if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
935  RISBG.Input.getOpcode() != ISD::TRUNCATE)
936  Count += 1;
937  if (Count == 0)
938  return false;
939 
940  // Prefer to use normal shift instructions over RISBG, since they can handle
941  // all cases and are sometimes shorter.
942  if (Count == 1 && N->getOpcode() != ISD::AND)
943  return false;
944 
945  // Prefer register extensions like LLC over RISBG. Also prefer to start
946  // out with normal ANDs if one instruction would be enough. We can convert
947  // these ANDs into an RISBG later if a three-address instruction is useful.
948  if (RISBG.Rotate == 0) {
949  bool PreferAnd = false;
950  // Prefer AND for any 32-bit and-immediate operation.
951  if (VT == MVT::i32)
952  PreferAnd = true;
953  // As well as for any 64-bit operation that can be implemented via LLC(R),
954  // LLH(R), LLGT(R), or one of the and-immediate instructions.
955  else if (RISBG.Mask == 0xff ||
956  RISBG.Mask == 0xffff ||
957  RISBG.Mask == 0x7fffffff ||
958  SystemZ::isImmLF(~RISBG.Mask) ||
959  SystemZ::isImmHF(~RISBG.Mask))
960  PreferAnd = true;
961  // And likewise for the LLZRGF instruction, which doesn't have a register
962  // to register version.
963  else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
964  if (Load->getMemoryVT() == MVT::i32 &&
965  (Load->getExtensionType() == ISD::EXTLOAD ||
966  Load->getExtensionType() == ISD::ZEXTLOAD) &&
967  RISBG.Mask == 0xffffff00 &&
968  Subtarget->hasLoadAndZeroRightmostByte())
969  PreferAnd = true;
970  }
971  if (PreferAnd) {
972  // Replace the current node with an AND. Note that the current node
973  // might already be that same AND, in which case it is already CSE'd
974  // with it, and we must not call ReplaceNode.
975  SDValue In = convertTo(DL, VT, RISBG.Input);
976  SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
977  SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
978  if (N != New.getNode()) {
979  insertDAGNode(CurDAG, N, Mask);
980  insertDAGNode(CurDAG, N, New);
981  ReplaceNode(N, New.getNode());
982  N = New.getNode();
983  }
984  // Now, select the machine opcode to implement this operation.
985  SelectCode(N);
986  return true;
987  }
988  }
989 
990  unsigned Opcode = SystemZ::RISBG;
991  // Prefer RISBGN if available, since it does not clobber CC.
992  if (Subtarget->hasMiscellaneousExtensions())
993  Opcode = SystemZ::RISBGN;
994  EVT OpcodeVT = MVT::i64;
995  if (VT == MVT::i32 && Subtarget->hasHighWord() &&
996  // We can only use the 32-bit instructions if all source bits are
997  // in the low 32 bits without wrapping, both after rotation (because
998  // of the smaller range for Start and End) and before rotation
999  // (because the input value is truncated).
1000  RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1001  ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1002  ((RISBG.End + RISBG.Rotate) & 63) >=
1003  ((RISBG.Start + RISBG.Rotate) & 63)) {
1004  Opcode = SystemZ::RISBMux;
1005  OpcodeVT = MVT::i32;
1006  RISBG.Start &= 31;
1007  RISBG.End &= 31;
1008  }
1009  SDValue Ops[5] = {
1010  getUNDEF(DL, OpcodeVT),
1011  convertTo(DL, OpcodeVT, RISBG.Input),
1012  CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1013  CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1014  CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1015  };
1016  SDValue New = convertTo(
1017  DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1018  ReplaceUses(N, New.getNode());
1019  CurDAG->RemoveDeadNode(N);
1020  return true;
1021 }
1022 
1023 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1024  SDLoc DL(N);
1025  EVT VT = N->getValueType(0);
1026  if (!VT.isInteger() || VT.getSizeInBits() > 64)
1027  return false;
1028  // Try treating each operand of N as the second operand of the RxSBG
1029  // and see which goes deepest.
1030  RxSBGOperands RxSBG[] = {
1031  RxSBGOperands(Opcode, N->getOperand(0)),
1032  RxSBGOperands(Opcode, N->getOperand(1))
1033  };
1034  unsigned Count[] = { 0, 0 };
1035  for (unsigned I = 0; I < 2; ++I)
1036  while (expandRxSBG(RxSBG[I]))
1037  // The widening or narrowing is expected to be free.
1038  // Counting widening or narrowing as a saved operation will result in
1039  // preferring an R*SBG over a simple shift/logical instruction.
1040  if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1041  RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1042  Count[I] += 1;
1043 
1044  // Do nothing if neither operand is suitable.
1045  if (Count[0] == 0 && Count[1] == 0)
1046  return false;
1047 
1048  // Pick the deepest second operand.
1049  unsigned I = Count[0] > Count[1] ? 0 : 1;
1050  SDValue Op0 = N->getOperand(I ^ 1);
1051 
1052  // Prefer IC for character insertions from memory.
1053  if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1054  if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1055  if (Load->getMemoryVT() == MVT::i8)
1056  return false;
1057 
1058  // See whether we can avoid an AND in the first operand by converting
1059  // ROSBG to RISBG.
1060  if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1061  Opcode = SystemZ::RISBG;
1062  // Prefer RISBGN if available, since it does not clobber CC.
1063  if (Subtarget->hasMiscellaneousExtensions())
1064  Opcode = SystemZ::RISBGN;
1065  }
1066 
1067  SDValue Ops[5] = {
1068  convertTo(DL, MVT::i64, Op0),
1069  convertTo(DL, MVT::i64, RxSBG[I].Input),
1070  CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1071  CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1072  CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1073  };
1074  SDValue New = convertTo(
1075  DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1076  ReplaceNode(N, New.getNode());
1077  return true;
1078 }
1079 
1080 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1081  SDValue Op0, uint64_t UpperVal,
1082  uint64_t LowerVal) {
1083  EVT VT = Node->getValueType(0);
1084  SDLoc DL(Node);
1085  SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1086  if (Op0.getNode())
1087  Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1088 
1089  {
1090  // When we haven't passed in Op0, Upper will be a constant. In order to
1091  // prevent folding back to the large immediate in `Or = getNode(...)` we run
1092  // SelectCode first and end up with an opaque machine node. This means that
1093  // we need to use a handle to keep track of Upper in case it gets CSE'd by
1094  // SelectCode.
1095  //
1096  // Note that in the case where Op0 is passed in we could just call
1097  // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1098  // the handle at all, but it's fine to do it here.
1099  //
1100  // TODO: This is a pretty hacky way to do this. Can we do something that
1101  // doesn't require a two paragraph explanation?
1102  HandleSDNode Handle(Upper);
1103  SelectCode(Upper.getNode());
1104  Upper = Handle.getValue();
1105  }
1106 
1107  SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1108  SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1109 
1110  ReplaceUses(Node, Or.getNode());
1111  CurDAG->RemoveDeadNode(Node);
1112 
1113  SelectCode(Or.getNode());
1114 }
1115 
1116 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1117  SDValue ElemV = N->getOperand(2);
1118  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1119  if (!ElemN)
1120  return false;
1121 
1122  unsigned Elem = ElemN->getZExtValue();
1123  EVT VT = N->getValueType(0);
1124  if (Elem >= VT.getVectorNumElements())
1125  return false;
1126 
1127  auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1128  if (!Load || !Load->hasOneUse())
1129  return false;
1130  if (Load->getMemoryVT().getSizeInBits() !=
1131  Load->getValueType(0).getSizeInBits())
1132  return false;
1133 
1134  SDValue Base, Disp, Index;
1135  if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1137  return false;
1138 
1139  SDLoc DL(Load);
1140  SDValue Ops[] = {
1141  N->getOperand(0), Base, Disp, Index,
1142  CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1143  };
1144  SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1145  ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1146  ReplaceNode(N, Res);
1147  return true;
1148 }
1149 
1150 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1151  SDValue Value = Store->getValue();
1152  if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1153  return false;
1154  if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1155  return false;
1156 
1157  SDValue ElemV = Value.getOperand(1);
1158  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1159  if (!ElemN)
1160  return false;
1161 
1162  SDValue Vec = Value.getOperand(0);
1163  EVT VT = Vec.getValueType();
1164  unsigned Elem = ElemN->getZExtValue();
1165  if (Elem >= VT.getVectorNumElements())
1166  return false;
1167 
1168  SDValue Base, Disp, Index;
1169  if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1171  return false;
1172 
1173  SDLoc DL(Store);
1174  SDValue Ops[] = {
1175  Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1176  Store->getChain()
1177  };
1178  ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1179  return true;
1180 }
1181 
1182 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1183  LoadSDNode *Load) const {
1184  // Check that the two memory operands have the same size.
1185  if (Load->getMemoryVT() != Store->getMemoryVT())
1186  return false;
1187 
1188  // Volatility stops an access from being decomposed.
1189  if (Load->isVolatile() || Store->isVolatile())
1190  return false;
1191 
1192  // There's no chance of overlap if the load is invariant.
1193  if (Load->isInvariant() && Load->isDereferenceable())
1194  return true;
1195 
1196  // Otherwise we need to check whether there's an alias.
1197  const Value *V1 = Load->getMemOperand()->getValue();
1198  const Value *V2 = Store->getMemOperand()->getValue();
1199  if (!V1 || !V2)
1200  return false;
1201 
1202  // Reject equality.
1203  uint64_t Size = Load->getMemoryVT().getStoreSize();
1204  int64_t End1 = Load->getSrcValueOffset() + Size;
1205  int64_t End2 = Store->getSrcValueOffset() + Size;
1206  if (V1 == V2 && End1 == End2)
1207  return false;
1208 
1209  return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()),
1210  MemoryLocation(V2, End2, Store->getAAInfo()));
1211 }
1212 
1213 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1214  auto *Store = cast<StoreSDNode>(N);
1215  auto *Load = cast<LoadSDNode>(Store->getValue());
1216 
1217  // Prefer not to use MVC if either address can use ... RELATIVE LONG
1218  // instructions.
1219  uint64_t Size = Load->getMemoryVT().getStoreSize();
1220  if (Size > 1 && Size <= 8) {
1221  // Prefer LHRL, LRL and LGRL.
1222  if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1223  return false;
1224  // Prefer STHRL, STRL and STGRL.
1225  if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1226  return false;
1227  }
1228 
1229  return canUseBlockOperation(Store, Load);
1230 }
1231 
1232 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1233  unsigned I) const {
1234  auto *StoreA = cast<StoreSDNode>(N);
1235  auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1236  auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1237  return !LoadA->isVolatile() && canUseBlockOperation(StoreA, LoadB);
1238 }
1239 
1240 void SystemZDAGToDAGISel::Select(SDNode *Node) {
1241  // Dump information about the Node being selected
1242  DEBUG(errs() << "Selecting: "; Node->dump(CurDAG); errs() << "\n");
1243 
1244  // If we have a custom node, we already have selected!
1245  if (Node->isMachineOpcode()) {
1246  DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1247  Node->setNodeId(-1);
1248  return;
1249  }
1250 
1251  unsigned Opcode = Node->getOpcode();
1252  switch (Opcode) {
1253  case ISD::OR:
1254  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1255  if (tryRxSBG(Node, SystemZ::ROSBG))
1256  return;
1257  goto or_xor;
1258 
1259  case ISD::XOR:
1260  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1261  if (tryRxSBG(Node, SystemZ::RXSBG))
1262  return;
1263  // Fall through.
1264  or_xor:
1265  // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1266  // split the operation into two. If both operands here happen to be
1267  // constant, leave this to common code to optimize.
1268  if (Node->getValueType(0) == MVT::i64 &&
1269  Node->getOperand(0).getOpcode() != ISD::Constant)
1270  if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1271  uint64_t Val = Op1->getZExtValue();
1272  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1273  splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1274  Val - uint32_t(Val), uint32_t(Val));
1275  return;
1276  }
1277  }
1278  break;
1279 
1280  case ISD::AND:
1281  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1282  if (tryRxSBG(Node, SystemZ::RNSBG))
1283  return;
1285  case ISD::ROTL:
1286  case ISD::SHL:
1287  case ISD::SRL:
1288  case ISD::ZERO_EXTEND:
1289  if (tryRISBGZero(Node))
1290  return;
1291  break;
1292 
1293  case ISD::Constant:
1294  // If this is a 64-bit constant that is out of the range of LLILF,
1295  // LLIHF and LGFI, split it into two 32-bit pieces.
1296  if (Node->getValueType(0) == MVT::i64) {
1297  uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1298  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1299  splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1300  uint32_t(Val));
1301  return;
1302  }
1303  }
1304  break;
1305 
1307  SDValue Op0 = Node->getOperand(0);
1308  SDValue Op1 = Node->getOperand(1);
1309  // Prefer to put any load first, so that it can be matched as a
1310  // conditional load. Likewise for constants in range for LOCHI.
1311  if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1312  (Subtarget->hasLoadStoreOnCond2() &&
1313  Node->getValueType(0).isInteger() &&
1314  Op1.getOpcode() == ISD::Constant &&
1315  isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1316  !(Op0.getOpcode() == ISD::Constant &&
1317  isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1318  SDValue CCValid = Node->getOperand(2);
1319  SDValue CCMask = Node->getOperand(3);
1320  uint64_t ConstCCValid =
1321  cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1322  uint64_t ConstCCMask =
1323  cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1324  // Invert the condition.
1325  CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node),
1326  CCMask.getValueType());
1327  SDValue Op4 = Node->getOperand(4);
1328  Node = CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1329  }
1330  break;
1331  }
1332 
1333  case ISD::INSERT_VECTOR_ELT: {
1334  EVT VT = Node->getValueType(0);
1335  unsigned ElemBitSize = VT.getScalarSizeInBits();
1336  if (ElemBitSize == 32) {
1337  if (tryGather(Node, SystemZ::VGEF))
1338  return;
1339  } else if (ElemBitSize == 64) {
1340  if (tryGather(Node, SystemZ::VGEG))
1341  return;
1342  }
1343  break;
1344  }
1345 
1346  case ISD::STORE: {
1347  auto *Store = cast<StoreSDNode>(Node);
1348  unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1349  if (ElemBitSize == 32) {
1350  if (tryScatter(Store, SystemZ::VSCEF))
1351  return;
1352  } else if (ElemBitSize == 64) {
1353  if (tryScatter(Store, SystemZ::VSCEG))
1354  return;
1355  }
1356  break;
1357  }
1358  }
1359 
1360  SelectCode(Node);
1361 }
1362 
1363 bool SystemZDAGToDAGISel::
1364 SelectInlineAsmMemoryOperand(const SDValue &Op,
1365  unsigned ConstraintID,
1366  std::vector<SDValue> &OutOps) {
1367  SystemZAddressingMode::AddrForm Form;
1368  SystemZAddressingMode::DispRange DispRange;
1369  SDValue Base, Disp, Index;
1370 
1371  switch(ConstraintID) {
1372  default:
1373  llvm_unreachable("Unexpected asm memory constraint");
1376  // Accept an address with a short displacement, but no index.
1377  Form = SystemZAddressingMode::FormBD;
1378  DispRange = SystemZAddressingMode::Disp12Only;
1379  break;
1381  // Accept an address with a short displacement and an index.
1382  Form = SystemZAddressingMode::FormBDXNormal;
1383  DispRange = SystemZAddressingMode::Disp12Only;
1384  break;
1386  // Accept an address with a long displacement, but no index.
1387  Form = SystemZAddressingMode::FormBD;
1388  DispRange = SystemZAddressingMode::Disp20Only;
1389  break;
1393  // Accept an address with a long displacement and an index.
1394  // m works the same as T, as this is the most general case.
1395  // We don't really have any special handling of "offsettable"
1396  // memory addresses, so just treat o the same as m.
1397  Form = SystemZAddressingMode::FormBDXNormal;
1398  DispRange = SystemZAddressingMode::Disp20Only;
1399  break;
1400  }
1401 
1402  if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1403  const TargetRegisterClass *TRC =
1404  Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1405  SDLoc DL(Base);
1406  SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1407 
1408  // Make sure that the base address doesn't go into %r0.
1409  // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1410  if (Base.getOpcode() != ISD::TargetFrameIndex &&
1411  Base.getOpcode() != ISD::Register) {
1412  Base =
1413  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1414  DL, Base.getValueType(),
1415  Base, RC), 0);
1416  }
1417 
1418  // Make sure that the index register isn't assigned to %r0 either.
1419  if (Index.getOpcode() != ISD::Register) {
1420  Index =
1421  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1422  DL, Index.getValueType(),
1423  Index, RC), 0);
1424  }
1425 
1426  OutOps.push_back(Base);
1427  OutOps.push_back(Disp);
1428  OutOps.push_back(Index);
1429  return false;
1430  }
1431 
1432  return true;
1433 }
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:170
bool isInvariant() const
EVT getValueType() const
Return the ValueType of the referenced return value.
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1542
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:165
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
const SDValue & getValue() const
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
const SDValue & getChain() const
constexpr bool isInt< 16 >(int64_t x)
Definition: MathExtras.h:298
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
void setNodeId(int Id)
Set unique node id.
SDNode * getNode() const
get the SDNode which holds the desired result
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static uint64_t allOnes(unsigned int Count)
bool hasOneUse() const
Return true if there is exactly one use of this node.
const HexagonInstrInfo * TII
Shift and rotation operations.
Definition: ISDOpcodes.h:379
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
unsigned getID() const
Return the register class ID number.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
const RegList & Regs
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
int64_t getSrcValueOffset() const
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
This class is used to represent ISD::STORE nodes.
const Value * getValue() const
Return the base address of the memory access.
const SDValue & getBasePtr() const
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
const SDValue & getOperand(unsigned Num) const
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:307
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
static const unsigned End
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
self_iterator getIterator()
Definition: ilist_node.h:82
bool isPCREL(unsigned Opcode)
Extended Value Type.
Definition: ValueTypes.h:34
bool isVolatile() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const SystemZInstrInfo * getInstrInfo() const override
Representation for a specific memory location.
void dump() const
Dump this node, for debugging.
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:301
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:314
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:209
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:96
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
bool isDereferenceable() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Target - Wrapper for Target specific information.
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:445
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:448
int getNodeId() const
Return the unique node id.
const SDValue & getValue() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:362
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:463
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:581
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
unsigned getOpcode() const
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
LLVM Value Representation.
Definition: Value.h:73
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:235
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
#define DEBUG(X)
Definition: Debug.h:118
bool isRxSBGMask(uint64_t Mask, unsigned BitSize, unsigned &Start, unsigned &End) const
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
Conversion operators.
Definition: ISDOpcodes.h:442
const SDValue & getOperand(unsigned i) const
uint64_t getZExtValue() const
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:451
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
This class is used to represent ISD::LOAD nodes.