LLVM  4.0.0
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines an instruction selector for the SystemZ target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SystemZTargetMachine.h"
17 #include "llvm/Support/Debug.h"
19 
20 using namespace llvm;
21 
22 #define DEBUG_TYPE "systemz-isel"
23 
24 namespace {
25 // Used to build addressing modes.
26 struct SystemZAddressingMode {
27  // The shape of the address.
28  enum AddrForm {
29  // base+displacement
30  FormBD,
31 
32  // base+displacement+index for load and store operands
33  FormBDXNormal,
34 
35  // base+displacement+index for load address operands
36  FormBDXLA,
37 
38  // base+displacement+index+ADJDYNALLOC
39  FormBDXDynAlloc
40  };
41  AddrForm Form;
42 
43  // The type of displacement. The enum names here correspond directly
44  // to the definitions in SystemZOperand.td. We could split them into
45  // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
46  enum DispRange {
47  Disp12Only,
48  Disp12Pair,
49  Disp20Only,
50  Disp20Only128,
51  Disp20Pair
52  };
53  DispRange DR;
54 
55  // The parts of the address. The address is equivalent to:
56  //
57  // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
58  SDValue Base;
59  int64_t Disp;
60  SDValue Index;
61  bool IncludesDynAlloc;
62 
63  SystemZAddressingMode(AddrForm form, DispRange dr)
64  : Form(form), DR(dr), Base(), Disp(0), Index(),
65  IncludesDynAlloc(false) {}
66 
67  // True if the address can have an index register.
68  bool hasIndexField() { return Form != FormBD; }
69 
70  // True if the address can (and must) include ADJDYNALLOC.
71  bool isDynAlloc() { return Form == FormBDXDynAlloc; }
72 
73  void dump() {
74  errs() << "SystemZAddressingMode " << this << '\n';
75 
76  errs() << " Base ";
77  if (Base.getNode())
78  Base.getNode()->dump();
79  else
80  errs() << "null\n";
81 
82  if (hasIndexField()) {
83  errs() << " Index ";
84  if (Index.getNode())
85  Index.getNode()->dump();
86  else
87  errs() << "null\n";
88  }
89 
90  errs() << " Disp " << Disp;
91  if (IncludesDynAlloc)
92  errs() << " + ADJDYNALLOC";
93  errs() << '\n';
94  }
95 };
96 
97 // Return a mask with Count low bits set.
98 static uint64_t allOnes(unsigned int Count) {
99  assert(Count <= 64);
100  if (Count > 63)
101  return UINT64_MAX;
102  return (uint64_t(1) << Count) - 1;
103 }
104 
105 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
106 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
107 // Rotate (I5). The combined operand value is effectively:
108 //
109 // (or (rotl Input, Rotate), ~Mask)
110 //
111 // for RNSBG and:
112 //
113 // (and (rotl Input, Rotate), Mask)
114 //
115 // otherwise. The output value has BitSize bits, although Input may be
116 // narrower (in which case the upper bits are don't care), or wider (in which
117 // case the result will be truncated as part of the operation).
118 struct RxSBGOperands {
119  RxSBGOperands(unsigned Op, SDValue N)
120  : Opcode(Op), BitSize(N.getValueSizeInBits()),
121  Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
122  Rotate(0) {}
123 
124  unsigned Opcode;
125  unsigned BitSize;
126  uint64_t Mask;
127  SDValue Input;
128  unsigned Start;
129  unsigned End;
130  unsigned Rotate;
131 };
132 
133 class SystemZDAGToDAGISel : public SelectionDAGISel {
134  const SystemZSubtarget *Subtarget;
135 
136  // Used by SystemZOperands.td to create integer constants.
137  inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
138  return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
139  }
140 
141  const SystemZTargetMachine &getTargetMachine() const {
142  return static_cast<const SystemZTargetMachine &>(TM);
143  }
144 
145  const SystemZInstrInfo *getInstrInfo() const {
146  return Subtarget->getInstrInfo();
147  }
148 
149  // Try to fold more of the base or index of AM into AM, where IsBase
150  // selects between the base and index.
151  bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
152 
153  // Try to describe N in AM, returning true on success.
154  bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
155 
156  // Extract individual target operands from matched address AM.
157  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
158  SDValue &Base, SDValue &Disp) const;
159  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
160  SDValue &Base, SDValue &Disp, SDValue &Index) const;
161 
162  // Try to match Addr as a FormBD address with displacement type DR.
163  // Return true on success, storing the base and displacement in
164  // Base and Disp respectively.
165  bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
166  SDValue &Base, SDValue &Disp) const;
167 
168  // Try to match Addr as a FormBDX address with displacement type DR.
169  // Return true on success and if the result had no index. Store the
170  // base and displacement in Base and Disp respectively.
171  bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
172  SDValue &Base, SDValue &Disp) const;
173 
174  // Try to match Addr as a FormBDX* address of form Form with
175  // displacement type DR. Return true on success, storing the base,
176  // displacement and index in Base, Disp and Index respectively.
177  bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
178  SystemZAddressingMode::DispRange DR, SDValue Addr,
179  SDValue &Base, SDValue &Disp, SDValue &Index) const;
180 
181  // PC-relative address matching routines used by SystemZOperands.td.
182  bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
183  if (SystemZISD::isPCREL(Addr.getOpcode())) {
184  Target = Addr.getOperand(0);
185  return true;
186  }
187  return false;
188  }
189 
190  // BD matching routines used by SystemZOperands.td.
191  bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
192  return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
193  }
194  bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
195  return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
196  }
197  bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
198  return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
199  }
200  bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
201  return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
202  }
203 
204  // MVI matching routines used by SystemZOperands.td.
205  bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
206  return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
207  }
208  bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
209  return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
210  }
211 
212  // BDX matching routines used by SystemZOperands.td.
213  bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
214  SDValue &Index) const {
215  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
216  SystemZAddressingMode::Disp12Only,
217  Addr, Base, Disp, Index);
218  }
219  bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
220  SDValue &Index) const {
221  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
222  SystemZAddressingMode::Disp12Pair,
223  Addr, Base, Disp, Index);
224  }
225  bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
226  SDValue &Index) const {
227  return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
228  SystemZAddressingMode::Disp12Only,
229  Addr, Base, Disp, Index);
230  }
231  bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
232  SDValue &Index) const {
233  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
234  SystemZAddressingMode::Disp20Only,
235  Addr, Base, Disp, Index);
236  }
237  bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
238  SDValue &Index) const {
239  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
240  SystemZAddressingMode::Disp20Only128,
241  Addr, Base, Disp, Index);
242  }
243  bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
244  SDValue &Index) const {
245  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
246  SystemZAddressingMode::Disp20Pair,
247  Addr, Base, Disp, Index);
248  }
249  bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
250  SDValue &Index) const {
251  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
252  SystemZAddressingMode::Disp12Pair,
253  Addr, Base, Disp, Index);
254  }
255  bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
256  SDValue &Index) const {
257  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
258  SystemZAddressingMode::Disp20Pair,
259  Addr, Base, Disp, Index);
260  }
261 
262  // Try to match Addr as an address with a base, 12-bit displacement
263  // and index, where the index is element Elem of a vector.
264  // Return true on success, storing the base, displacement and vector
265  // in Base, Disp and Index respectively.
266  bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
267  SDValue &Disp, SDValue &Index) const;
268 
269  // Check whether (or Op (and X InsertMask)) is effectively an insertion
270  // of X into bits InsertMask of some Y != Op. Return true if so and
271  // set Op to that Y.
272  bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
273 
274  // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
275  // Return true on success.
276  bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
277 
278  // Try to fold some of RxSBG.Input into other fields of RxSBG.
279  // Return true on success.
280  bool expandRxSBG(RxSBGOperands &RxSBG) const;
281 
282  // Return an undefined value of type VT.
283  SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
284 
285  // Convert N to VT, if it isn't already.
286  SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
287 
288  // Try to implement AND or shift node N using RISBG with the zero flag set.
289  // Return the selected node on success, otherwise return null.
290  bool tryRISBGZero(SDNode *N);
291 
292  // Try to use RISBG or Opcode to implement OR or XOR node N.
293  // Return the selected node on success, otherwise return null.
294  bool tryRxSBG(SDNode *N, unsigned Opcode);
295 
296  // If Op0 is null, then Node is a constant that can be loaded using:
297  //
298  // (Opcode UpperVal LowerVal)
299  //
300  // If Op0 is nonnull, then Node can be implemented using:
301  //
302  // (Opcode (Opcode Op0 UpperVal) LowerVal)
303  void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
304  uint64_t UpperVal, uint64_t LowerVal);
305 
306  // Try to use gather instruction Opcode to implement vector insertion N.
307  bool tryGather(SDNode *N, unsigned Opcode);
308 
309  // Try to use scatter instruction Opcode to implement store Store.
310  bool tryScatter(StoreSDNode *Store, unsigned Opcode);
311 
312  // Return true if Load and Store are loads and stores of the same size
313  // and are guaranteed not to overlap. Such operations can be implemented
314  // using block (SS-format) instructions.
315  //
316  // Partial overlap would lead to incorrect code, since the block operations
317  // are logically bytewise, even though they have a fast path for the
318  // non-overlapping case. We also need to avoid full overlap (i.e. two
319  // addresses that might be equal at run time) because although that case
320  // would be handled correctly, it might be implemented by millicode.
321  bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
322 
323  // N is a (store (load Y), X) pattern. Return true if it can use an MVC
324  // from Y to X.
325  bool storeLoadCanUseMVC(SDNode *N) const;
326 
327  // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
328  // if A[1 - I] == X and if N can use a block operation like NC from A[I]
329  // to X.
330  bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
331 
332 public:
333  SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
334  : SelectionDAGISel(TM, OptLevel) {}
335 
336  bool runOnMachineFunction(MachineFunction &MF) override {
337  Subtarget = &MF.getSubtarget<SystemZSubtarget>();
339  }
340 
341  // Override MachineFunctionPass.
342  StringRef getPassName() const override {
343  return "SystemZ DAG->DAG Pattern Instruction Selection";
344  }
345 
346  // Override SelectionDAGISel.
347  void Select(SDNode *Node) override;
348  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
349  std::vector<SDValue> &OutOps) override;
350 
351  // Include the pieces autogenerated from the target description.
352  #include "SystemZGenDAGISel.inc"
353 };
354 } // end anonymous namespace
355 
357  CodeGenOpt::Level OptLevel) {
358  return new SystemZDAGToDAGISel(TM, OptLevel);
359 }
360 
361 // Return true if Val should be selected as a displacement for an address
362 // with range DR. Here we're interested in the range of both the instruction
363 // described by DR and of any pairing instruction.
364 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
365  switch (DR) {
366  case SystemZAddressingMode::Disp12Only:
367  return isUInt<12>(Val);
368 
369  case SystemZAddressingMode::Disp12Pair:
370  case SystemZAddressingMode::Disp20Only:
371  case SystemZAddressingMode::Disp20Pair:
372  return isInt<20>(Val);
373 
374  case SystemZAddressingMode::Disp20Only128:
375  return isInt<20>(Val) && isInt<20>(Val + 8);
376  }
377  llvm_unreachable("Unhandled displacement range");
378 }
379 
380 // Change the base or index in AM to Value, where IsBase selects
381 // between the base and index.
382 static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
383  SDValue Value) {
384  if (IsBase)
385  AM.Base = Value;
386  else
387  AM.Index = Value;
388 }
389 
390 // The base or index of AM is equivalent to Value + ADJDYNALLOC,
391 // where IsBase selects between the base and index. Try to fold the
392 // ADJDYNALLOC into AM.
393 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
394  SDValue Value) {
395  if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
396  changeComponent(AM, IsBase, Value);
397  AM.IncludesDynAlloc = true;
398  return true;
399  }
400  return false;
401 }
402 
403 // The base of AM is equivalent to Base + Index. Try to use Index as
404 // the index register.
405 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
406  SDValue Index) {
407  if (AM.hasIndexField() && !AM.Index.getNode()) {
408  AM.Base = Base;
409  AM.Index = Index;
410  return true;
411  }
412  return false;
413 }
414 
415 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
416 // between the base and index. Try to fold Op1 into AM's displacement.
417 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
418  SDValue Op0, uint64_t Op1) {
419  // First try adjusting the displacement.
420  int64_t TestDisp = AM.Disp + Op1;
421  if (selectDisp(AM.DR, TestDisp)) {
422  changeComponent(AM, IsBase, Op0);
423  AM.Disp = TestDisp;
424  return true;
425  }
426 
427  // We could consider forcing the displacement into a register and
428  // using it as an index, but it would need to be carefully tuned.
429  return false;
430 }
431 
432 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
433  bool IsBase) const {
434  SDValue N = IsBase ? AM.Base : AM.Index;
435  unsigned Opcode = N.getOpcode();
436  if (Opcode == ISD::TRUNCATE) {
437  N = N.getOperand(0);
438  Opcode = N.getOpcode();
439  }
440  if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
441  SDValue Op0 = N.getOperand(0);
442  SDValue Op1 = N.getOperand(1);
443 
444  unsigned Op0Code = Op0->getOpcode();
445  unsigned Op1Code = Op1->getOpcode();
446 
447  if (Op0Code == SystemZISD::ADJDYNALLOC)
448  return expandAdjDynAlloc(AM, IsBase, Op1);
449  if (Op1Code == SystemZISD::ADJDYNALLOC)
450  return expandAdjDynAlloc(AM, IsBase, Op0);
451 
452  if (Op0Code == ISD::Constant)
453  return expandDisp(AM, IsBase, Op1,
454  cast<ConstantSDNode>(Op0)->getSExtValue());
455  if (Op1Code == ISD::Constant)
456  return expandDisp(AM, IsBase, Op0,
457  cast<ConstantSDNode>(Op1)->getSExtValue());
458 
459  if (IsBase && expandIndex(AM, Op0, Op1))
460  return true;
461  }
462  if (Opcode == SystemZISD::PCREL_OFFSET) {
463  SDValue Full = N.getOperand(0);
464  SDValue Base = N.getOperand(1);
465  SDValue Anchor = Base.getOperand(0);
466  uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
467  cast<GlobalAddressSDNode>(Anchor)->getOffset());
468  return expandDisp(AM, IsBase, Base, Offset);
469  }
470  return false;
471 }
472 
473 // Return true if an instruction with displacement range DR should be
474 // used for displacement value Val. selectDisp(DR, Val) must already hold.
475 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
476  assert(selectDisp(DR, Val) && "Invalid displacement");
477  switch (DR) {
478  case SystemZAddressingMode::Disp12Only:
479  case SystemZAddressingMode::Disp20Only:
480  case SystemZAddressingMode::Disp20Only128:
481  return true;
482 
483  case SystemZAddressingMode::Disp12Pair:
484  // Use the other instruction if the displacement is too large.
485  return isUInt<12>(Val);
486 
487  case SystemZAddressingMode::Disp20Pair:
488  // Use the other instruction if the displacement is small enough.
489  return !isUInt<12>(Val);
490  }
491  llvm_unreachable("Unhandled displacement range");
492 }
493 
494 // Return true if Base + Disp + Index should be performed by LA(Y).
495 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
496  // Don't use LA(Y) for constants.
497  if (!Base)
498  return false;
499 
500  // Always use LA(Y) for frame addresses, since we know that the destination
501  // register is almost always (perhaps always) going to be different from
502  // the frame register.
503  if (Base->getOpcode() == ISD::FrameIndex)
504  return true;
505 
506  if (Disp) {
507  // Always use LA(Y) if there is a base, displacement and index.
508  if (Index)
509  return true;
510 
511  // Always use LA if the displacement is small enough. It should always
512  // be no worse than AGHI (and better if it avoids a move).
513  if (isUInt<12>(Disp))
514  return true;
515 
516  // For similar reasons, always use LAY if the constant is too big for AGHI.
517  // LAY should be no worse than AGFI.
518  if (!isInt<16>(Disp))
519  return true;
520  } else {
521  // Don't use LA for plain registers.
522  if (!Index)
523  return false;
524 
525  // Don't use LA for plain addition if the index operand is only used
526  // once. It should be a natural two-operand addition in that case.
527  if (Index->hasOneUse())
528  return false;
529 
530  // Prefer addition if the second operation is sign-extended, in the
531  // hope of using AGF.
532  unsigned IndexOpcode = Index->getOpcode();
533  if (IndexOpcode == ISD::SIGN_EXTEND ||
534  IndexOpcode == ISD::SIGN_EXTEND_INREG)
535  return false;
536  }
537 
538  // Don't use LA for two-operand addition if either operand is only
539  // used once. The addition instructions are better in that case.
540  if (Base->hasOneUse())
541  return false;
542 
543  return true;
544 }
545 
546 // Return true if Addr is suitable for AM, updating AM if so.
547 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
548  SystemZAddressingMode &AM) const {
549  // Start out assuming that the address will need to be loaded separately,
550  // then try to extend it as much as we can.
551  AM.Base = Addr;
552 
553  // First try treating the address as a constant.
554  if (Addr.getOpcode() == ISD::Constant &&
555  expandDisp(AM, true, SDValue(),
556  cast<ConstantSDNode>(Addr)->getSExtValue()))
557  ;
558  // Also see if it's a bare ADJDYNALLOC.
559  else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
560  expandAdjDynAlloc(AM, true, SDValue()))
561  ;
562  else
563  // Otherwise try expanding each component.
564  while (expandAddress(AM, true) ||
565  (AM.Index.getNode() && expandAddress(AM, false)))
566  continue;
567 
568  // Reject cases where it isn't profitable to use LA(Y).
569  if (AM.Form == SystemZAddressingMode::FormBDXLA &&
570  !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
571  return false;
572 
573  // Reject cases where the other instruction in a pair should be used.
574  if (!isValidDisp(AM.DR, AM.Disp))
575  return false;
576 
577  // Make sure that ADJDYNALLOC is included where necessary.
578  if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
579  return false;
580 
581  DEBUG(AM.dump());
582  return true;
583 }
584 
585 // Insert a node into the DAG at least before Pos. This will reposition
586 // the node as needed, and will assign it a node ID that is <= Pos's ID.
587 // Note that this does *not* preserve the uniqueness of node IDs!
588 // The selection DAG must no longer depend on their uniqueness when this
589 // function is used.
590 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
591  if (N.getNode()->getNodeId() == -1 ||
592  N.getNode()->getNodeId() > Pos->getNodeId()) {
593  DAG->RepositionNode(Pos->getIterator(), N.getNode());
594  N.getNode()->setNodeId(Pos->getNodeId());
595  }
596 }
597 
598 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
599  EVT VT, SDValue &Base,
600  SDValue &Disp) const {
601  Base = AM.Base;
602  if (!Base.getNode())
603  // Register 0 means "no base". This is mostly useful for shifts.
604  Base = CurDAG->getRegister(0, VT);
605  else if (Base.getOpcode() == ISD::FrameIndex) {
606  // Lower a FrameIndex to a TargetFrameIndex.
607  int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
608  Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
609  } else if (Base.getValueType() != VT) {
610  // Truncate values from i64 to i32, for shifts.
611  assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
612  "Unexpected truncation");
613  SDLoc DL(Base);
614  SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
615  insertDAGNode(CurDAG, Base.getNode(), Trunc);
616  Base = Trunc;
617  }
618 
619  // Lower the displacement to a TargetConstant.
620  Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
621 }
622 
623 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
624  EVT VT, SDValue &Base,
625  SDValue &Disp,
626  SDValue &Index) const {
627  getAddressOperands(AM, VT, Base, Disp);
628 
629  Index = AM.Index;
630  if (!Index.getNode())
631  // Register 0 means "no index".
632  Index = CurDAG->getRegister(0, VT);
633 }
634 
635 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
636  SDValue Addr, SDValue &Base,
637  SDValue &Disp) const {
638  SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
639  if (!selectAddress(Addr, AM))
640  return false;
641 
642  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
643  return true;
644 }
645 
646 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
647  SDValue Addr, SDValue &Base,
648  SDValue &Disp) const {
649  SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
650  if (!selectAddress(Addr, AM) || AM.Index.getNode())
651  return false;
652 
653  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
654  return true;
655 }
656 
657 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
658  SystemZAddressingMode::DispRange DR,
659  SDValue Addr, SDValue &Base,
660  SDValue &Disp, SDValue &Index) const {
661  SystemZAddressingMode AM(Form, DR);
662  if (!selectAddress(Addr, AM))
663  return false;
664 
665  getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
666  return true;
667 }
668 
669 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
670  SDValue &Base,
671  SDValue &Disp,
672  SDValue &Index) const {
673  SDValue Regs[2];
674  if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
675  Regs[0].getNode() && Regs[1].getNode()) {
676  for (unsigned int I = 0; I < 2; ++I) {
677  Base = Regs[I];
678  Index = Regs[1 - I];
679  // We can't tell here whether the index vector has the right type
680  // for the access; the caller needs to do that instead.
681  if (Index.getOpcode() == ISD::ZERO_EXTEND)
682  Index = Index.getOperand(0);
683  if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
684  Index.getOperand(1) == Elem) {
685  Index = Index.getOperand(0);
686  return true;
687  }
688  }
689  }
690  return false;
691 }
692 
693 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
694  uint64_t InsertMask) const {
695  // We're only interested in cases where the insertion is into some operand
696  // of Op, rather than into Op itself. The only useful case is an AND.
697  if (Op.getOpcode() != ISD::AND)
698  return false;
699 
700  // We need a constant mask.
701  auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
702  if (!MaskNode)
703  return false;
704 
705  // It's not an insertion of Op.getOperand(0) if the two masks overlap.
706  uint64_t AndMask = MaskNode->getZExtValue();
707  if (InsertMask & AndMask)
708  return false;
709 
710  // It's only an insertion if all bits are covered or are known to be zero.
711  // The inner check covers all cases but is more expensive.
712  uint64_t Used = allOnes(Op.getValueSizeInBits());
713  if (Used != (AndMask | InsertMask)) {
714  APInt KnownZero, KnownOne;
715  CurDAG->computeKnownBits(Op.getOperand(0), KnownZero, KnownOne);
716  if (Used != (AndMask | InsertMask | KnownZero.getZExtValue()))
717  return false;
718  }
719 
720  Op = Op.getOperand(0);
721  return true;
722 }
723 
724 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
725  uint64_t Mask) const {
726  const SystemZInstrInfo *TII = getInstrInfo();
727  if (RxSBG.Rotate != 0)
728  Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
729  Mask &= RxSBG.Mask;
730  if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
731  RxSBG.Mask = Mask;
732  return true;
733  }
734  return false;
735 }
736 
737 // Return true if any bits of (RxSBG.Input & Mask) are significant.
738 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
739  // Rotate the mask in the same way as RxSBG.Input is rotated.
740  if (RxSBG.Rotate != 0)
741  Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
742  return (Mask & RxSBG.Mask) != 0;
743 }
744 
745 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
746  SDValue N = RxSBG.Input;
747  unsigned Opcode = N.getOpcode();
748  switch (Opcode) {
749  case ISD::TRUNCATE: {
750  if (RxSBG.Opcode == SystemZ::RNSBG)
751  return false;
752  uint64_t BitSize = N.getValueSizeInBits();
753  uint64_t Mask = allOnes(BitSize);
754  if (!refineRxSBGMask(RxSBG, Mask))
755  return false;
756  RxSBG.Input = N.getOperand(0);
757  return true;
758  }
759  case ISD::AND: {
760  if (RxSBG.Opcode == SystemZ::RNSBG)
761  return false;
762 
763  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
764  if (!MaskNode)
765  return false;
766 
767  SDValue Input = N.getOperand(0);
768  uint64_t Mask = MaskNode->getZExtValue();
769  if (!refineRxSBGMask(RxSBG, Mask)) {
770  // If some bits of Input are already known zeros, those bits will have
771  // been removed from the mask. See if adding them back in makes the
772  // mask suitable.
773  APInt KnownZero, KnownOne;
774  CurDAG->computeKnownBits(Input, KnownZero, KnownOne);
775  Mask |= KnownZero.getZExtValue();
776  if (!refineRxSBGMask(RxSBG, Mask))
777  return false;
778  }
779  RxSBG.Input = Input;
780  return true;
781  }
782 
783  case ISD::OR: {
784  if (RxSBG.Opcode != SystemZ::RNSBG)
785  return false;
786 
787  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
788  if (!MaskNode)
789  return false;
790 
791  SDValue Input = N.getOperand(0);
792  uint64_t Mask = ~MaskNode->getZExtValue();
793  if (!refineRxSBGMask(RxSBG, Mask)) {
794  // If some bits of Input are already known ones, those bits will have
795  // been removed from the mask. See if adding them back in makes the
796  // mask suitable.
797  APInt KnownZero, KnownOne;
798  CurDAG->computeKnownBits(Input, KnownZero, KnownOne);
799  Mask &= ~KnownOne.getZExtValue();
800  if (!refineRxSBGMask(RxSBG, Mask))
801  return false;
802  }
803  RxSBG.Input = Input;
804  return true;
805  }
806 
807  case ISD::ROTL: {
808  // Any 64-bit rotate left can be merged into the RxSBG.
809  if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
810  return false;
811  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
812  if (!CountNode)
813  return false;
814 
815  RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
816  RxSBG.Input = N.getOperand(0);
817  return true;
818  }
819 
820  case ISD::ANY_EXTEND:
821  // Bits above the extended operand are don't-care.
822  RxSBG.Input = N.getOperand(0);
823  return true;
824 
825  case ISD::ZERO_EXTEND:
826  if (RxSBG.Opcode != SystemZ::RNSBG) {
827  // Restrict the mask to the extended operand.
828  unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
829  if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
830  return false;
831 
832  RxSBG.Input = N.getOperand(0);
833  return true;
834  }
836 
837  case ISD::SIGN_EXTEND: {
838  // Check that the extension bits are don't-care (i.e. are masked out
839  // by the final mask).
840  unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
841  if (maskMatters(RxSBG, allOnes(RxSBG.BitSize) - allOnes(InnerBitSize)))
842  return false;
843 
844  RxSBG.Input = N.getOperand(0);
845  return true;
846  }
847 
848  case ISD::SHL: {
849  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
850  if (!CountNode)
851  return false;
852 
853  uint64_t Count = CountNode->getZExtValue();
854  unsigned BitSize = N.getValueSizeInBits();
855  if (Count < 1 || Count >= BitSize)
856  return false;
857 
858  if (RxSBG.Opcode == SystemZ::RNSBG) {
859  // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
860  // count bits from RxSBG.Input are ignored.
861  if (maskMatters(RxSBG, allOnes(Count)))
862  return false;
863  } else {
864  // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
865  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
866  return false;
867  }
868 
869  RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
870  RxSBG.Input = N.getOperand(0);
871  return true;
872  }
873 
874  case ISD::SRL:
875  case ISD::SRA: {
876  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
877  if (!CountNode)
878  return false;
879 
880  uint64_t Count = CountNode->getZExtValue();
881  unsigned BitSize = N.getValueSizeInBits();
882  if (Count < 1 || Count >= BitSize)
883  return false;
884 
885  if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
886  // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
887  // count bits from RxSBG.Input are ignored.
888  if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
889  return false;
890  } else {
891  // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
892  // which is similar to SLL above.
893  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
894  return false;
895  }
896 
897  RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
898  RxSBG.Input = N.getOperand(0);
899  return true;
900  }
901  default:
902  return false;
903  }
904 }
905 
906 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
907  SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
908  return SDValue(N, 0);
909 }
910 
911 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
912  SDValue N) const {
913  if (N.getValueType() == MVT::i32 && VT == MVT::i64)
914  return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
915  DL, VT, getUNDEF(DL, MVT::i64), N);
916  if (N.getValueType() == MVT::i64 && VT == MVT::i32)
917  return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
918  assert(N.getValueType() == VT && "Unexpected value types");
919  return N;
920 }
921 
922 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
923  SDLoc DL(N);
924  EVT VT = N->getValueType(0);
925  if (!VT.isInteger() || VT.getSizeInBits() > 64)
926  return false;
927  RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
928  unsigned Count = 0;
929  while (expandRxSBG(RISBG))
930  // The widening or narrowing is expected to be free.
931  // Counting widening or narrowing as a saved operation will result in
932  // preferring an R*SBG over a simple shift/logical instruction.
933  if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
934  RISBG.Input.getOpcode() != ISD::TRUNCATE)
935  Count += 1;
936  if (Count == 0)
937  return false;
938 
939  // Prefer to use normal shift instructions over RISBG, since they can handle
940  // all cases and are sometimes shorter.
941  if (Count == 1 && N->getOpcode() != ISD::AND)
942  return false;
943 
944  // Prefer register extensions like LLC over RISBG. Also prefer to start
945  // out with normal ANDs if one instruction would be enough. We can convert
946  // these ANDs into an RISBG later if a three-address instruction is useful.
947  if (RISBG.Rotate == 0) {
948  bool PreferAnd = false;
949  // Prefer AND for any 32-bit and-immediate operation.
950  if (VT == MVT::i32)
951  PreferAnd = true;
952  // As well as for any 64-bit operation that can be implemented via LLC(R),
953  // LLH(R), LLGT(R), or one of the and-immediate instructions.
954  else if (RISBG.Mask == 0xff ||
955  RISBG.Mask == 0xffff ||
956  RISBG.Mask == 0x7fffffff ||
957  SystemZ::isImmLF(~RISBG.Mask) ||
958  SystemZ::isImmHF(~RISBG.Mask))
959  PreferAnd = true;
960  // And likewise for the LLZRGF instruction, which doesn't have a register
961  // to register version.
962  else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
963  if (Load->getMemoryVT() == MVT::i32 &&
964  (Load->getExtensionType() == ISD::EXTLOAD ||
965  Load->getExtensionType() == ISD::ZEXTLOAD) &&
966  RISBG.Mask == 0xffffff00 &&
967  Subtarget->hasLoadAndZeroRightmostByte())
968  PreferAnd = true;
969  }
970  if (PreferAnd) {
971  // Replace the current node with an AND. Note that the current node
972  // might already be that same AND, in which case it is already CSE'd
973  // with it, and we must not call ReplaceNode.
974  SDValue In = convertTo(DL, VT, RISBG.Input);
975  SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
976  SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
977  if (N != New.getNode()) {
978  insertDAGNode(CurDAG, N, Mask);
979  insertDAGNode(CurDAG, N, New);
980  ReplaceNode(N, New.getNode());
981  N = New.getNode();
982  }
983  // Now, select the machine opcode to implement this operation.
984  SelectCode(N);
985  return true;
986  }
987  }
988 
989  unsigned Opcode = SystemZ::RISBG;
990  // Prefer RISBGN if available, since it does not clobber CC.
991  if (Subtarget->hasMiscellaneousExtensions())
992  Opcode = SystemZ::RISBGN;
993  EVT OpcodeVT = MVT::i64;
994  if (VT == MVT::i32 && Subtarget->hasHighWord()) {
995  Opcode = SystemZ::RISBMux;
996  OpcodeVT = MVT::i32;
997  RISBG.Start &= 31;
998  RISBG.End &= 31;
999  }
1000  SDValue Ops[5] = {
1001  getUNDEF(DL, OpcodeVT),
1002  convertTo(DL, OpcodeVT, RISBG.Input),
1003  CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1004  CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1005  CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1006  };
1007  SDValue New = convertTo(
1008  DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1009  ReplaceUses(N, New.getNode());
1010  CurDAG->RemoveDeadNode(N);
1011  return true;
1012 }
1013 
1014 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1015  SDLoc DL(N);
1016  EVT VT = N->getValueType(0);
1017  if (!VT.isInteger() || VT.getSizeInBits() > 64)
1018  return false;
1019  // Try treating each operand of N as the second operand of the RxSBG
1020  // and see which goes deepest.
1021  RxSBGOperands RxSBG[] = {
1022  RxSBGOperands(Opcode, N->getOperand(0)),
1023  RxSBGOperands(Opcode, N->getOperand(1))
1024  };
1025  unsigned Count[] = { 0, 0 };
1026  for (unsigned I = 0; I < 2; ++I)
1027  while (expandRxSBG(RxSBG[I]))
1028  // The widening or narrowing is expected to be free.
1029  // Counting widening or narrowing as a saved operation will result in
1030  // preferring an R*SBG over a simple shift/logical instruction.
1031  if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1032  RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1033  Count[I] += 1;
1034 
1035  // Do nothing if neither operand is suitable.
1036  if (Count[0] == 0 && Count[1] == 0)
1037  return false;
1038 
1039  // Pick the deepest second operand.
1040  unsigned I = Count[0] > Count[1] ? 0 : 1;
1041  SDValue Op0 = N->getOperand(I ^ 1);
1042 
1043  // Prefer IC for character insertions from memory.
1044  if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1045  if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1046  if (Load->getMemoryVT() == MVT::i8)
1047  return false;
1048 
1049  // See whether we can avoid an AND in the first operand by converting
1050  // ROSBG to RISBG.
1051  if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1052  Opcode = SystemZ::RISBG;
1053  // Prefer RISBGN if available, since it does not clobber CC.
1054  if (Subtarget->hasMiscellaneousExtensions())
1055  Opcode = SystemZ::RISBGN;
1056  }
1057 
1058  SDValue Ops[5] = {
1059  convertTo(DL, MVT::i64, Op0),
1060  convertTo(DL, MVT::i64, RxSBG[I].Input),
1061  CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1062  CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1063  CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1064  };
1065  SDValue New = convertTo(
1066  DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1067  ReplaceNode(N, New.getNode());
1068  return true;
1069 }
1070 
1071 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1072  SDValue Op0, uint64_t UpperVal,
1073  uint64_t LowerVal) {
1074  EVT VT = Node->getValueType(0);
1075  SDLoc DL(Node);
1076  SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1077  if (Op0.getNode())
1078  Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1079 
1080  {
1081  // When we haven't passed in Op0, Upper will be a constant. In order to
1082  // prevent folding back to the large immediate in `Or = getNode(...)` we run
1083  // SelectCode first and end up with an opaque machine node. This means that
1084  // we need to use a handle to keep track of Upper in case it gets CSE'd by
1085  // SelectCode.
1086  //
1087  // Note that in the case where Op0 is passed in we could just call
1088  // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1089  // the handle at all, but it's fine to do it here.
1090  //
1091  // TODO: This is a pretty hacky way to do this. Can we do something that
1092  // doesn't require a two paragraph explanation?
1093  HandleSDNode Handle(Upper);
1094  SelectCode(Upper.getNode());
1095  Upper = Handle.getValue();
1096  }
1097 
1098  SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1099  SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1100 
1101  ReplaceUses(Node, Or.getNode());
1102  CurDAG->RemoveDeadNode(Node);
1103 
1104  SelectCode(Or.getNode());
1105 }
1106 
1107 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1108  SDValue ElemV = N->getOperand(2);
1109  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1110  if (!ElemN)
1111  return false;
1112 
1113  unsigned Elem = ElemN->getZExtValue();
1114  EVT VT = N->getValueType(0);
1115  if (Elem >= VT.getVectorNumElements())
1116  return false;
1117 
1118  auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1119  if (!Load || !Load->hasOneUse())
1120  return false;
1121  if (Load->getMemoryVT().getSizeInBits() !=
1122  Load->getValueType(0).getSizeInBits())
1123  return false;
1124 
1125  SDValue Base, Disp, Index;
1126  if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1128  return false;
1129 
1130  SDLoc DL(Load);
1131  SDValue Ops[] = {
1132  N->getOperand(0), Base, Disp, Index,
1133  CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1134  };
1135  SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1136  ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1137  ReplaceNode(N, Res);
1138  return true;
1139 }
1140 
1141 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1142  SDValue Value = Store->getValue();
1143  if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1144  return false;
1145  if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1146  return false;
1147 
1148  SDValue ElemV = Value.getOperand(1);
1149  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1150  if (!ElemN)
1151  return false;
1152 
1153  SDValue Vec = Value.getOperand(0);
1154  EVT VT = Vec.getValueType();
1155  unsigned Elem = ElemN->getZExtValue();
1156  if (Elem >= VT.getVectorNumElements())
1157  return false;
1158 
1159  SDValue Base, Disp, Index;
1160  if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1162  return false;
1163 
1164  SDLoc DL(Store);
1165  SDValue Ops[] = {
1166  Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1167  Store->getChain()
1168  };
1169  ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1170  return true;
1171 }
1172 
1173 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1174  LoadSDNode *Load) const {
1175  // Check that the two memory operands have the same size.
1176  if (Load->getMemoryVT() != Store->getMemoryVT())
1177  return false;
1178 
1179  // Volatility stops an access from being decomposed.
1180  if (Load->isVolatile() || Store->isVolatile())
1181  return false;
1182 
1183  // There's no chance of overlap if the load is invariant.
1184  if (Load->isInvariant() && Load->isDereferenceable())
1185  return true;
1186 
1187  // Otherwise we need to check whether there's an alias.
1188  const Value *V1 = Load->getMemOperand()->getValue();
1189  const Value *V2 = Store->getMemOperand()->getValue();
1190  if (!V1 || !V2)
1191  return false;
1192 
1193  // Reject equality.
1194  uint64_t Size = Load->getMemoryVT().getStoreSize();
1195  int64_t End1 = Load->getSrcValueOffset() + Size;
1196  int64_t End2 = Store->getSrcValueOffset() + Size;
1197  if (V1 == V2 && End1 == End2)
1198  return false;
1199 
1200  return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()),
1201  MemoryLocation(V2, End2, Store->getAAInfo()));
1202 }
1203 
1204 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1205  auto *Store = cast<StoreSDNode>(N);
1206  auto *Load = cast<LoadSDNode>(Store->getValue());
1207 
1208  // Prefer not to use MVC if either address can use ... RELATIVE LONG
1209  // instructions.
1210  uint64_t Size = Load->getMemoryVT().getStoreSize();
1211  if (Size > 1 && Size <= 8) {
1212  // Prefer LHRL, LRL and LGRL.
1213  if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1214  return false;
1215  // Prefer STHRL, STRL and STGRL.
1216  if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1217  return false;
1218  }
1219 
1220  return canUseBlockOperation(Store, Load);
1221 }
1222 
1223 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1224  unsigned I) const {
1225  auto *StoreA = cast<StoreSDNode>(N);
1226  auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1227  auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1228  return !LoadA->isVolatile() && canUseBlockOperation(StoreA, LoadB);
1229 }
1230 
1231 void SystemZDAGToDAGISel::Select(SDNode *Node) {
1232  // Dump information about the Node being selected
1233  DEBUG(errs() << "Selecting: "; Node->dump(CurDAG); errs() << "\n");
1234 
1235  // If we have a custom node, we already have selected!
1236  if (Node->isMachineOpcode()) {
1237  DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1238  Node->setNodeId(-1);
1239  return;
1240  }
1241 
1242  unsigned Opcode = Node->getOpcode();
1243  switch (Opcode) {
1244  case ISD::OR:
1245  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1246  if (tryRxSBG(Node, SystemZ::ROSBG))
1247  return;
1248  goto or_xor;
1249 
1250  case ISD::XOR:
1251  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1252  if (tryRxSBG(Node, SystemZ::RXSBG))
1253  return;
1254  // Fall through.
1255  or_xor:
1256  // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1257  // split the operation into two.
1258  if (Node->getValueType(0) == MVT::i64)
1259  if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1260  uint64_t Val = Op1->getZExtValue();
1261  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1262  splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1263  Val - uint32_t(Val), uint32_t(Val));
1264  return;
1265  }
1266  }
1267  break;
1268 
1269  case ISD::AND:
1270  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1271  if (tryRxSBG(Node, SystemZ::RNSBG))
1272  return;
1274  case ISD::ROTL:
1275  case ISD::SHL:
1276  case ISD::SRL:
1277  case ISD::ZERO_EXTEND:
1278  if (tryRISBGZero(Node))
1279  return;
1280  break;
1281 
1282  case ISD::Constant:
1283  // If this is a 64-bit constant that is out of the range of LLILF,
1284  // LLIHF and LGFI, split it into two 32-bit pieces.
1285  if (Node->getValueType(0) == MVT::i64) {
1286  uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1287  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1288  splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1289  uint32_t(Val));
1290  return;
1291  }
1292  }
1293  break;
1294 
1296  SDValue Op0 = Node->getOperand(0);
1297  SDValue Op1 = Node->getOperand(1);
1298  // Prefer to put any load first, so that it can be matched as a
1299  // conditional load. Likewise for constants in range for LOCHI.
1300  if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1301  (Subtarget->hasLoadStoreOnCond2() &&
1302  Node->getValueType(0).isInteger() &&
1303  Op1.getOpcode() == ISD::Constant &&
1304  isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1305  !(Op0.getOpcode() == ISD::Constant &&
1306  isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1307  SDValue CCValid = Node->getOperand(2);
1308  SDValue CCMask = Node->getOperand(3);
1309  uint64_t ConstCCValid =
1310  cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1311  uint64_t ConstCCMask =
1312  cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1313  // Invert the condition.
1314  CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node),
1315  CCMask.getValueType());
1316  SDValue Op4 = Node->getOperand(4);
1317  Node = CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1318  }
1319  break;
1320  }
1321 
1322  case ISD::INSERT_VECTOR_ELT: {
1323  EVT VT = Node->getValueType(0);
1324  unsigned ElemBitSize = VT.getScalarSizeInBits();
1325  if (ElemBitSize == 32) {
1326  if (tryGather(Node, SystemZ::VGEF))
1327  return;
1328  } else if (ElemBitSize == 64) {
1329  if (tryGather(Node, SystemZ::VGEG))
1330  return;
1331  }
1332  break;
1333  }
1334 
1335  case ISD::STORE: {
1336  auto *Store = cast<StoreSDNode>(Node);
1337  unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1338  if (ElemBitSize == 32) {
1339  if (tryScatter(Store, SystemZ::VSCEF))
1340  return;
1341  } else if (ElemBitSize == 64) {
1342  if (tryScatter(Store, SystemZ::VSCEG))
1343  return;
1344  }
1345  break;
1346  }
1347  }
1348 
1349  SelectCode(Node);
1350 }
1351 
1352 bool SystemZDAGToDAGISel::
1353 SelectInlineAsmMemoryOperand(const SDValue &Op,
1354  unsigned ConstraintID,
1355  std::vector<SDValue> &OutOps) {
1356  SystemZAddressingMode::AddrForm Form;
1357  SystemZAddressingMode::DispRange DispRange;
1358  SDValue Base, Disp, Index;
1359 
1360  switch(ConstraintID) {
1361  default:
1362  llvm_unreachable("Unexpected asm memory constraint");
1365  // Accept an address with a short displacement, but no index.
1366  Form = SystemZAddressingMode::FormBD;
1367  DispRange = SystemZAddressingMode::Disp12Only;
1368  break;
1370  // Accept an address with a short displacement and an index.
1371  Form = SystemZAddressingMode::FormBDXNormal;
1372  DispRange = SystemZAddressingMode::Disp12Only;
1373  break;
1375  // Accept an address with a long displacement, but no index.
1376  Form = SystemZAddressingMode::FormBD;
1377  DispRange = SystemZAddressingMode::Disp20Only;
1378  break;
1381  // Accept an address with a long displacement and an index.
1382  // m works the same as T, as this is the most general case.
1383  Form = SystemZAddressingMode::FormBDXNormal;
1384  DispRange = SystemZAddressingMode::Disp20Only;
1385  break;
1386  }
1387 
1388  if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1389  const TargetRegisterClass *TRC =
1390  Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1391  SDLoc DL(Base);
1392  SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1393 
1394  // Make sure that the base address doesn't go into %r0.
1395  // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1396  if (Base.getOpcode() != ISD::TargetFrameIndex &&
1397  Base.getOpcode() != ISD::Register) {
1398  Base =
1399  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1400  DL, Base.getValueType(),
1401  Base, RC), 0);
1402  }
1403 
1404  // Make sure that the index register isn't assigned to %r0 either.
1405  if (Index.getOpcode() != ISD::Register) {
1406  Index =
1407  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1408  DL, Index.getValueType(),
1409  Index, RC), 0);
1410  }
1411 
1412  OutOps.push_back(Base);
1413  OutOps.push_back(Disp);
1414  OutOps.push_back(Index);
1415  return false;
1416  }
1417 
1418  return true;
1419 }
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:170
SDValue getValue(unsigned R) const
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1309
bool hasOneUse() const
Return true if there is exactly one use of this node.
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:165
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getID() const
Return the register class ID number.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
constexpr bool isInt< 16 >(int64_t x)
Definition: MathExtras.h:271
const SDValue & getOperand(unsigned Num) const
void setNodeId(int Id)
Set unique node id.
int64_t getSrcValueOffset() const
static uint64_t allOnes(unsigned int Count)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const HexagonInstrInfo * TII
Shift and rotation operations.
Definition: ISDOpcodes.h:344
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:123
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const RegList & Regs
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
const SDValue & getBasePtr() const
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
EVT getMemoryVT() const
Return the type of the in-memory value.
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
This class is used to represent ISD::STORE nodes.
SDNode * getNode() const
get the SDNode which holds the desired result
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:262
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:268
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
const SDValue & getOperand(unsigned i) const
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:279
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1947
uint32_t Offset
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
static const unsigned End
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
self_iterator getIterator()
Definition: ilist_node.h:81
bool isPCREL(unsigned Opcode)
bool isVolatile() const
const SDValue & getValue() const
bool isRxSBGMask(uint64_t Mask, unsigned BitSize, unsigned &Start, unsigned &End) const
EVT - Extended Value Type.
Definition: ValueTypes.h:31
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Representation for a specific memory location.
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:274
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:285
bool isInvariant() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:166
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
bool isDereferenceable() const
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Target - Wrapper for Target specific information.
Class for arbitrary precision integers.
Definition: APInt.h:77
const Value * getValue() const
Return the base address of the memory access.
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:400
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:403
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:333
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:418
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:536
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
int getNodeId() const
Return the unique node id.
EVT getValueType() const
Return the ValueType of the referenced return value.
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
LLVM Value Representation.
Definition: Value.h:71
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:239
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
#define DEBUG(X)
Definition: Debug.h:100
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
Conversion operators.
Definition: ISDOpcodes.h:397
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:406
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
EVT changeVectorElementTypeToInteger() const
changeVectorElementTypeToInteger - Return a vector with the same number of elements as this vector...
Definition: ValueTypes.h:80
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
uint64_t getZExtValue() const
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:248
This class is used to represent ISD::LOAD nodes.