LLVM  3.7.0
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines an instruction selector for the SystemZ target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SystemZTargetMachine.h"
17 #include "llvm/Support/Debug.h"
19 
20 using namespace llvm;
21 
22 #define DEBUG_TYPE "systemz-isel"
23 
24 namespace {
25 // Used to build addressing modes.
26 struct SystemZAddressingMode {
27  // The shape of the address.
28  enum AddrForm {
29  // base+displacement
30  FormBD,
31 
32  // base+displacement+index for load and store operands
33  FormBDXNormal,
34 
35  // base+displacement+index for load address operands
36  FormBDXLA,
37 
38  // base+displacement+index+ADJDYNALLOC
39  FormBDXDynAlloc
40  };
41  AddrForm Form;
42 
43  // The type of displacement. The enum names here correspond directly
44  // to the definitions in SystemZOperand.td. We could split them into
45  // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
46  enum DispRange {
47  Disp12Only,
48  Disp12Pair,
49  Disp20Only,
50  Disp20Only128,
51  Disp20Pair
52  };
53  DispRange DR;
54 
55  // The parts of the address. The address is equivalent to:
56  //
57  // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
58  SDValue Base;
59  int64_t Disp;
60  SDValue Index;
61  bool IncludesDynAlloc;
62 
63  SystemZAddressingMode(AddrForm form, DispRange dr)
64  : Form(form), DR(dr), Base(), Disp(0), Index(),
65  IncludesDynAlloc(false) {}
66 
67  // True if the address can have an index register.
68  bool hasIndexField() { return Form != FormBD; }
69 
70  // True if the address can (and must) include ADJDYNALLOC.
71  bool isDynAlloc() { return Form == FormBDXDynAlloc; }
72 
73  void dump() {
74  errs() << "SystemZAddressingMode " << this << '\n';
75 
76  errs() << " Base ";
77  if (Base.getNode())
78  Base.getNode()->dump();
79  else
80  errs() << "null\n";
81 
82  if (hasIndexField()) {
83  errs() << " Index ";
84  if (Index.getNode())
85  Index.getNode()->dump();
86  else
87  errs() << "null\n";
88  }
89 
90  errs() << " Disp " << Disp;
91  if (IncludesDynAlloc)
92  errs() << " + ADJDYNALLOC";
93  errs() << '\n';
94  }
95 };
96 
97 // Return a mask with Count low bits set.
98 static uint64_t allOnes(unsigned int Count) {
99  assert(Count <= 64);
100  if (Count > 63)
101  return UINT64_MAX;
102  return (uint64_t(1) << Count) - 1;
103 }
104 
105 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
106 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
107 // Rotate (I5). The combined operand value is effectively:
108 //
109 // (or (rotl Input, Rotate), ~Mask)
110 //
111 // for RNSBG and:
112 //
113 // (and (rotl Input, Rotate), Mask)
114 //
115 // otherwise. The output value has BitSize bits, although Input may be
116 // narrower (in which case the upper bits are don't care).
117 struct RxSBGOperands {
118  RxSBGOperands(unsigned Op, SDValue N)
119  : Opcode(Op), BitSize(N.getValueType().getSizeInBits()),
120  Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
121  Rotate(0) {}
122 
123  unsigned Opcode;
124  unsigned BitSize;
125  uint64_t Mask;
126  SDValue Input;
127  unsigned Start;
128  unsigned End;
129  unsigned Rotate;
130 };
131 
132 class SystemZDAGToDAGISel : public SelectionDAGISel {
133  const SystemZSubtarget *Subtarget;
134 
135  // Used by SystemZOperands.td to create integer constants.
136  inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
137  return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
138  }
139 
140  const SystemZTargetMachine &getTargetMachine() const {
141  return static_cast<const SystemZTargetMachine &>(TM);
142  }
143 
144  const SystemZInstrInfo *getInstrInfo() const {
145  return Subtarget->getInstrInfo();
146  }
147 
148  // Try to fold more of the base or index of AM into AM, where IsBase
149  // selects between the base and index.
150  bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
151 
152  // Try to describe N in AM, returning true on success.
153  bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
154 
155  // Extract individual target operands from matched address AM.
156  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
157  SDValue &Base, SDValue &Disp) const;
158  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159  SDValue &Base, SDValue &Disp, SDValue &Index) const;
160 
161  // Try to match Addr as a FormBD address with displacement type DR.
162  // Return true on success, storing the base and displacement in
163  // Base and Disp respectively.
164  bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
165  SDValue &Base, SDValue &Disp) const;
166 
167  // Try to match Addr as a FormBDX address with displacement type DR.
168  // Return true on success and if the result had no index. Store the
169  // base and displacement in Base and Disp respectively.
170  bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
171  SDValue &Base, SDValue &Disp) const;
172 
173  // Try to match Addr as a FormBDX* address of form Form with
174  // displacement type DR. Return true on success, storing the base,
175  // displacement and index in Base, Disp and Index respectively.
176  bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
177  SystemZAddressingMode::DispRange DR, SDValue Addr,
178  SDValue &Base, SDValue &Disp, SDValue &Index) const;
179 
180  // PC-relative address matching routines used by SystemZOperands.td.
181  bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
182  if (SystemZISD::isPCREL(Addr.getOpcode())) {
183  Target = Addr.getOperand(0);
184  return true;
185  }
186  return false;
187  }
188 
189  // BD matching routines used by SystemZOperands.td.
190  bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
191  return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
192  }
193  bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
194  return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
195  }
196  bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
197  return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
198  }
199  bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
200  return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
201  }
202 
203  // MVI matching routines used by SystemZOperands.td.
204  bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
205  return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
206  }
207  bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
208  return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
209  }
210 
211  // BDX matching routines used by SystemZOperands.td.
212  bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
213  SDValue &Index) const {
214  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
215  SystemZAddressingMode::Disp12Only,
216  Addr, Base, Disp, Index);
217  }
218  bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
219  SDValue &Index) const {
220  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
221  SystemZAddressingMode::Disp12Pair,
222  Addr, Base, Disp, Index);
223  }
224  bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
225  SDValue &Index) const {
226  return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
227  SystemZAddressingMode::Disp12Only,
228  Addr, Base, Disp, Index);
229  }
230  bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
231  SDValue &Index) const {
232  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
233  SystemZAddressingMode::Disp20Only,
234  Addr, Base, Disp, Index);
235  }
236  bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
237  SDValue &Index) const {
238  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
239  SystemZAddressingMode::Disp20Only128,
240  Addr, Base, Disp, Index);
241  }
242  bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
243  SDValue &Index) const {
244  return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
245  SystemZAddressingMode::Disp20Pair,
246  Addr, Base, Disp, Index);
247  }
248  bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
249  SDValue &Index) const {
250  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
251  SystemZAddressingMode::Disp12Pair,
252  Addr, Base, Disp, Index);
253  }
254  bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
255  SDValue &Index) const {
256  return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
257  SystemZAddressingMode::Disp20Pair,
258  Addr, Base, Disp, Index);
259  }
260 
261  // Try to match Addr as an address with a base, 12-bit displacement
262  // and index, where the index is element Elem of a vector.
263  // Return true on success, storing the base, displacement and vector
264  // in Base, Disp and Index respectively.
265  bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
266  SDValue &Disp, SDValue &Index) const;
267 
268  // Check whether (or Op (and X InsertMask)) is effectively an insertion
269  // of X into bits InsertMask of some Y != Op. Return true if so and
270  // set Op to that Y.
271  bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
272 
273  // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
274  // Return true on success.
275  bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
276 
277  // Try to fold some of RxSBG.Input into other fields of RxSBG.
278  // Return true on success.
279  bool expandRxSBG(RxSBGOperands &RxSBG) const;
280 
281  // Return an undefined value of type VT.
282  SDValue getUNDEF(SDLoc DL, EVT VT) const;
283 
284  // Convert N to VT, if it isn't already.
285  SDValue convertTo(SDLoc DL, EVT VT, SDValue N) const;
286 
287  // Try to implement AND or shift node N using RISBG with the zero flag set.
288  // Return the selected node on success, otherwise return null.
289  SDNode *tryRISBGZero(SDNode *N);
290 
291  // Try to use RISBG or Opcode to implement OR or XOR node N.
292  // Return the selected node on success, otherwise return null.
293  SDNode *tryRxSBG(SDNode *N, unsigned Opcode);
294 
295  // If Op0 is null, then Node is a constant that can be loaded using:
296  //
297  // (Opcode UpperVal LowerVal)
298  //
299  // If Op0 is nonnull, then Node can be implemented using:
300  //
301  // (Opcode (Opcode Op0 UpperVal) LowerVal)
302  SDNode *splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
303  uint64_t UpperVal, uint64_t LowerVal);
304 
305  // Try to use gather instruction Opcode to implement vector insertion N.
306  SDNode *tryGather(SDNode *N, unsigned Opcode);
307 
308  // Try to use scatter instruction Opcode to implement store Store.
309  SDNode *tryScatter(StoreSDNode *Store, unsigned Opcode);
310 
311  // Return true if Load and Store are loads and stores of the same size
312  // and are guaranteed not to overlap. Such operations can be implemented
313  // using block (SS-format) instructions.
314  //
315  // Partial overlap would lead to incorrect code, since the block operations
316  // are logically bytewise, even though they have a fast path for the
317  // non-overlapping case. We also need to avoid full overlap (i.e. two
318  // addresses that might be equal at run time) because although that case
319  // would be handled correctly, it might be implemented by millicode.
320  bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
321 
322  // N is a (store (load Y), X) pattern. Return true if it can use an MVC
323  // from Y to X.
324  bool storeLoadCanUseMVC(SDNode *N) const;
325 
326  // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
327  // if A[1 - I] == X and if N can use a block operation like NC from A[I]
328  // to X.
329  bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
330 
331 public:
332  SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
333  : SelectionDAGISel(TM, OptLevel) {}
334 
335  bool runOnMachineFunction(MachineFunction &MF) override {
336  Subtarget = &MF.getSubtarget<SystemZSubtarget>();
338  }
339 
340  // Override MachineFunctionPass.
341  const char *getPassName() const override {
342  return "SystemZ DAG->DAG Pattern Instruction Selection";
343  }
344 
345  // Override SelectionDAGISel.
346  SDNode *Select(SDNode *Node) override;
347  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
348  std::vector<SDValue> &OutOps) override;
349 
350  // Include the pieces autogenerated from the target description.
351  #include "SystemZGenDAGISel.inc"
352 };
353 } // end anonymous namespace
354 
356  CodeGenOpt::Level OptLevel) {
357  return new SystemZDAGToDAGISel(TM, OptLevel);
358 }
359 
360 // Return true if Val should be selected as a displacement for an address
361 // with range DR. Here we're interested in the range of both the instruction
362 // described by DR and of any pairing instruction.
363 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
364  switch (DR) {
365  case SystemZAddressingMode::Disp12Only:
366  return isUInt<12>(Val);
367 
368  case SystemZAddressingMode::Disp12Pair:
369  case SystemZAddressingMode::Disp20Only:
370  case SystemZAddressingMode::Disp20Pair:
371  return isInt<20>(Val);
372 
373  case SystemZAddressingMode::Disp20Only128:
374  return isInt<20>(Val) && isInt<20>(Val + 8);
375  }
376  llvm_unreachable("Unhandled displacement range");
377 }
378 
379 // Change the base or index in AM to Value, where IsBase selects
380 // between the base and index.
381 static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
382  SDValue Value) {
383  if (IsBase)
384  AM.Base = Value;
385  else
386  AM.Index = Value;
387 }
388 
389 // The base or index of AM is equivalent to Value + ADJDYNALLOC,
390 // where IsBase selects between the base and index. Try to fold the
391 // ADJDYNALLOC into AM.
392 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
393  SDValue Value) {
394  if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
395  changeComponent(AM, IsBase, Value);
396  AM.IncludesDynAlloc = true;
397  return true;
398  }
399  return false;
400 }
401 
402 // The base of AM is equivalent to Base + Index. Try to use Index as
403 // the index register.
404 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
405  SDValue Index) {
406  if (AM.hasIndexField() && !AM.Index.getNode()) {
407  AM.Base = Base;
408  AM.Index = Index;
409  return true;
410  }
411  return false;
412 }
413 
414 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
415 // between the base and index. Try to fold Op1 into AM's displacement.
416 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
417  SDValue Op0, uint64_t Op1) {
418  // First try adjusting the displacement.
419  int64_t TestDisp = AM.Disp + Op1;
420  if (selectDisp(AM.DR, TestDisp)) {
421  changeComponent(AM, IsBase, Op0);
422  AM.Disp = TestDisp;
423  return true;
424  }
425 
426  // We could consider forcing the displacement into a register and
427  // using it as an index, but it would need to be carefully tuned.
428  return false;
429 }
430 
431 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
432  bool IsBase) const {
433  SDValue N = IsBase ? AM.Base : AM.Index;
434  unsigned Opcode = N.getOpcode();
435  if (Opcode == ISD::TRUNCATE) {
436  N = N.getOperand(0);
437  Opcode = N.getOpcode();
438  }
439  if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
440  SDValue Op0 = N.getOperand(0);
441  SDValue Op1 = N.getOperand(1);
442 
443  unsigned Op0Code = Op0->getOpcode();
444  unsigned Op1Code = Op1->getOpcode();
445 
446  if (Op0Code == SystemZISD::ADJDYNALLOC)
447  return expandAdjDynAlloc(AM, IsBase, Op1);
448  if (Op1Code == SystemZISD::ADJDYNALLOC)
449  return expandAdjDynAlloc(AM, IsBase, Op0);
450 
451  if (Op0Code == ISD::Constant)
452  return expandDisp(AM, IsBase, Op1,
453  cast<ConstantSDNode>(Op0)->getSExtValue());
454  if (Op1Code == ISD::Constant)
455  return expandDisp(AM, IsBase, Op0,
456  cast<ConstantSDNode>(Op1)->getSExtValue());
457 
458  if (IsBase && expandIndex(AM, Op0, Op1))
459  return true;
460  }
461  if (Opcode == SystemZISD::PCREL_OFFSET) {
462  SDValue Full = N.getOperand(0);
463  SDValue Base = N.getOperand(1);
464  SDValue Anchor = Base.getOperand(0);
465  uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
466  cast<GlobalAddressSDNode>(Anchor)->getOffset());
467  return expandDisp(AM, IsBase, Base, Offset);
468  }
469  return false;
470 }
471 
472 // Return true if an instruction with displacement range DR should be
473 // used for displacement value Val. selectDisp(DR, Val) must already hold.
474 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
475  assert(selectDisp(DR, Val) && "Invalid displacement");
476  switch (DR) {
477  case SystemZAddressingMode::Disp12Only:
478  case SystemZAddressingMode::Disp20Only:
479  case SystemZAddressingMode::Disp20Only128:
480  return true;
481 
482  case SystemZAddressingMode::Disp12Pair:
483  // Use the other instruction if the displacement is too large.
484  return isUInt<12>(Val);
485 
486  case SystemZAddressingMode::Disp20Pair:
487  // Use the other instruction if the displacement is small enough.
488  return !isUInt<12>(Val);
489  }
490  llvm_unreachable("Unhandled displacement range");
491 }
492 
493 // Return true if Base + Disp + Index should be performed by LA(Y).
494 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
495  // Don't use LA(Y) for constants.
496  if (!Base)
497  return false;
498 
499  // Always use LA(Y) for frame addresses, since we know that the destination
500  // register is almost always (perhaps always) going to be different from
501  // the frame register.
502  if (Base->getOpcode() == ISD::FrameIndex)
503  return true;
504 
505  if (Disp) {
506  // Always use LA(Y) if there is a base, displacement and index.
507  if (Index)
508  return true;
509 
510  // Always use LA if the displacement is small enough. It should always
511  // be no worse than AGHI (and better if it avoids a move).
512  if (isUInt<12>(Disp))
513  return true;
514 
515  // For similar reasons, always use LAY if the constant is too big for AGHI.
516  // LAY should be no worse than AGFI.
517  if (!isInt<16>(Disp))
518  return true;
519  } else {
520  // Don't use LA for plain registers.
521  if (!Index)
522  return false;
523 
524  // Don't use LA for plain addition if the index operand is only used
525  // once. It should be a natural two-operand addition in that case.
526  if (Index->hasOneUse())
527  return false;
528 
529  // Prefer addition if the second operation is sign-extended, in the
530  // hope of using AGF.
531  unsigned IndexOpcode = Index->getOpcode();
532  if (IndexOpcode == ISD::SIGN_EXTEND ||
533  IndexOpcode == ISD::SIGN_EXTEND_INREG)
534  return false;
535  }
536 
537  // Don't use LA for two-operand addition if either operand is only
538  // used once. The addition instructions are better in that case.
539  if (Base->hasOneUse())
540  return false;
541 
542  return true;
543 }
544 
545 // Return true if Addr is suitable for AM, updating AM if so.
546 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
547  SystemZAddressingMode &AM) const {
548  // Start out assuming that the address will need to be loaded separately,
549  // then try to extend it as much as we can.
550  AM.Base = Addr;
551 
552  // First try treating the address as a constant.
553  if (Addr.getOpcode() == ISD::Constant &&
554  expandDisp(AM, true, SDValue(),
555  cast<ConstantSDNode>(Addr)->getSExtValue()))
556  ;
557  else
558  // Otherwise try expanding each component.
559  while (expandAddress(AM, true) ||
560  (AM.Index.getNode() && expandAddress(AM, false)))
561  continue;
562 
563  // Reject cases where it isn't profitable to use LA(Y).
564  if (AM.Form == SystemZAddressingMode::FormBDXLA &&
565  !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
566  return false;
567 
568  // Reject cases where the other instruction in a pair should be used.
569  if (!isValidDisp(AM.DR, AM.Disp))
570  return false;
571 
572  // Make sure that ADJDYNALLOC is included where necessary.
573  if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
574  return false;
575 
576  DEBUG(AM.dump());
577  return true;
578 }
579 
580 // Insert a node into the DAG at least before Pos. This will reposition
581 // the node as needed, and will assign it a node ID that is <= Pos's ID.
582 // Note that this does *not* preserve the uniqueness of node IDs!
583 // The selection DAG must no longer depend on their uniqueness when this
584 // function is used.
585 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
586  if (N.getNode()->getNodeId() == -1 ||
587  N.getNode()->getNodeId() > Pos->getNodeId()) {
588  DAG->RepositionNode(Pos, N.getNode());
589  N.getNode()->setNodeId(Pos->getNodeId());
590  }
591 }
592 
593 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
594  EVT VT, SDValue &Base,
595  SDValue &Disp) const {
596  Base = AM.Base;
597  if (!Base.getNode())
598  // Register 0 means "no base". This is mostly useful for shifts.
599  Base = CurDAG->getRegister(0, VT);
600  else if (Base.getOpcode() == ISD::FrameIndex) {
601  // Lower a FrameIndex to a TargetFrameIndex.
602  int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
603  Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
604  } else if (Base.getValueType() != VT) {
605  // Truncate values from i64 to i32, for shifts.
606  assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
607  "Unexpected truncation");
608  SDLoc DL(Base);
609  SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
610  insertDAGNode(CurDAG, Base.getNode(), Trunc);
611  Base = Trunc;
612  }
613 
614  // Lower the displacement to a TargetConstant.
615  Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
616 }
617 
618 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
619  EVT VT, SDValue &Base,
620  SDValue &Disp,
621  SDValue &Index) const {
622  getAddressOperands(AM, VT, Base, Disp);
623 
624  Index = AM.Index;
625  if (!Index.getNode())
626  // Register 0 means "no index".
627  Index = CurDAG->getRegister(0, VT);
628 }
629 
630 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
631  SDValue Addr, SDValue &Base,
632  SDValue &Disp) const {
633  SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
634  if (!selectAddress(Addr, AM))
635  return false;
636 
637  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
638  return true;
639 }
640 
641 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
642  SDValue Addr, SDValue &Base,
643  SDValue &Disp) const {
644  SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
645  if (!selectAddress(Addr, AM) || AM.Index.getNode())
646  return false;
647 
648  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
649  return true;
650 }
651 
652 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
653  SystemZAddressingMode::DispRange DR,
654  SDValue Addr, SDValue &Base,
655  SDValue &Disp, SDValue &Index) const {
656  SystemZAddressingMode AM(Form, DR);
657  if (!selectAddress(Addr, AM))
658  return false;
659 
660  getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
661  return true;
662 }
663 
664 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
665  SDValue &Base,
666  SDValue &Disp,
667  SDValue &Index) const {
668  SDValue Regs[2];
669  if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
670  Regs[0].getNode() && Regs[1].getNode()) {
671  for (unsigned int I = 0; I < 2; ++I) {
672  Base = Regs[I];
673  Index = Regs[1 - I];
674  // We can't tell here whether the index vector has the right type
675  // for the access; the caller needs to do that instead.
676  if (Index.getOpcode() == ISD::ZERO_EXTEND)
677  Index = Index.getOperand(0);
678  if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
679  Index.getOperand(1) == Elem) {
680  Index = Index.getOperand(0);
681  return true;
682  }
683  }
684  }
685  return false;
686 }
687 
688 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
689  uint64_t InsertMask) const {
690  // We're only interested in cases where the insertion is into some operand
691  // of Op, rather than into Op itself. The only useful case is an AND.
692  if (Op.getOpcode() != ISD::AND)
693  return false;
694 
695  // We need a constant mask.
696  auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
697  if (!MaskNode)
698  return false;
699 
700  // It's not an insertion of Op.getOperand(0) if the two masks overlap.
701  uint64_t AndMask = MaskNode->getZExtValue();
702  if (InsertMask & AndMask)
703  return false;
704 
705  // It's only an insertion if all bits are covered or are known to be zero.
706  // The inner check covers all cases but is more expensive.
707  uint64_t Used = allOnes(Op.getValueType().getSizeInBits());
708  if (Used != (AndMask | InsertMask)) {
709  APInt KnownZero, KnownOne;
710  CurDAG->computeKnownBits(Op.getOperand(0), KnownZero, KnownOne);
711  if (Used != (AndMask | InsertMask | KnownZero.getZExtValue()))
712  return false;
713  }
714 
715  Op = Op.getOperand(0);
716  return true;
717 }
718 
719 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
720  uint64_t Mask) const {
721  const SystemZInstrInfo *TII = getInstrInfo();
722  if (RxSBG.Rotate != 0)
723  Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
724  Mask &= RxSBG.Mask;
725  if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
726  RxSBG.Mask = Mask;
727  return true;
728  }
729  return false;
730 }
731 
732 // Return true if any bits of (RxSBG.Input & Mask) are significant.
733 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
734  // Rotate the mask in the same way as RxSBG.Input is rotated.
735  if (RxSBG.Rotate != 0)
736  Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
737  return (Mask & RxSBG.Mask) != 0;
738 }
739 
740 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
741  SDValue N = RxSBG.Input;
742  unsigned Opcode = N.getOpcode();
743  switch (Opcode) {
744  case ISD::AND: {
745  if (RxSBG.Opcode == SystemZ::RNSBG)
746  return false;
747 
748  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
749  if (!MaskNode)
750  return false;
751 
752  SDValue Input = N.getOperand(0);
753  uint64_t Mask = MaskNode->getZExtValue();
754  if (!refineRxSBGMask(RxSBG, Mask)) {
755  // If some bits of Input are already known zeros, those bits will have
756  // been removed from the mask. See if adding them back in makes the
757  // mask suitable.
758  APInt KnownZero, KnownOne;
759  CurDAG->computeKnownBits(Input, KnownZero, KnownOne);
760  Mask |= KnownZero.getZExtValue();
761  if (!refineRxSBGMask(RxSBG, Mask))
762  return false;
763  }
764  RxSBG.Input = Input;
765  return true;
766  }
767 
768  case ISD::OR: {
769  if (RxSBG.Opcode != SystemZ::RNSBG)
770  return false;
771 
772  auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
773  if (!MaskNode)
774  return false;
775 
776  SDValue Input = N.getOperand(0);
777  uint64_t Mask = ~MaskNode->getZExtValue();
778  if (!refineRxSBGMask(RxSBG, Mask)) {
779  // If some bits of Input are already known ones, those bits will have
780  // been removed from the mask. See if adding them back in makes the
781  // mask suitable.
782  APInt KnownZero, KnownOne;
783  CurDAG->computeKnownBits(Input, KnownZero, KnownOne);
784  Mask &= ~KnownOne.getZExtValue();
785  if (!refineRxSBGMask(RxSBG, Mask))
786  return false;
787  }
788  RxSBG.Input = Input;
789  return true;
790  }
791 
792  case ISD::ROTL: {
793  // Any 64-bit rotate left can be merged into the RxSBG.
794  if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
795  return false;
796  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
797  if (!CountNode)
798  return false;
799 
800  RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
801  RxSBG.Input = N.getOperand(0);
802  return true;
803  }
804 
805  case ISD::ANY_EXTEND:
806  // Bits above the extended operand are don't-care.
807  RxSBG.Input = N.getOperand(0);
808  return true;
809 
810  case ISD::ZERO_EXTEND:
811  if (RxSBG.Opcode != SystemZ::RNSBG) {
812  // Restrict the mask to the extended operand.
813  unsigned InnerBitSize = N.getOperand(0).getValueType().getSizeInBits();
814  if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
815  return false;
816 
817  RxSBG.Input = N.getOperand(0);
818  return true;
819  }
820  // Fall through.
821 
822  case ISD::SIGN_EXTEND: {
823  // Check that the extension bits are don't-care (i.e. are masked out
824  // by the final mask).
825  unsigned InnerBitSize = N.getOperand(0).getValueType().getSizeInBits();
826  if (maskMatters(RxSBG, allOnes(RxSBG.BitSize) - allOnes(InnerBitSize)))
827  return false;
828 
829  RxSBG.Input = N.getOperand(0);
830  return true;
831  }
832 
833  case ISD::SHL: {
834  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
835  if (!CountNode)
836  return false;
837 
838  uint64_t Count = CountNode->getZExtValue();
839  unsigned BitSize = N.getValueType().getSizeInBits();
840  if (Count < 1 || Count >= BitSize)
841  return false;
842 
843  if (RxSBG.Opcode == SystemZ::RNSBG) {
844  // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
845  // count bits from RxSBG.Input are ignored.
846  if (maskMatters(RxSBG, allOnes(Count)))
847  return false;
848  } else {
849  // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
850  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
851  return false;
852  }
853 
854  RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
855  RxSBG.Input = N.getOperand(0);
856  return true;
857  }
858 
859  case ISD::SRL:
860  case ISD::SRA: {
861  auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
862  if (!CountNode)
863  return false;
864 
865  uint64_t Count = CountNode->getZExtValue();
866  unsigned BitSize = N.getValueType().getSizeInBits();
867  if (Count < 1 || Count >= BitSize)
868  return false;
869 
870  if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
871  // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
872  // count bits from RxSBG.Input are ignored.
873  if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
874  return false;
875  } else {
876  // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
877  // which is similar to SLL above.
878  if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
879  return false;
880  }
881 
882  RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
883  RxSBG.Input = N.getOperand(0);
884  return true;
885  }
886  default:
887  return false;
888  }
889 }
890 
891 SDValue SystemZDAGToDAGISel::getUNDEF(SDLoc DL, EVT VT) const {
892  SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
893  return SDValue(N, 0);
894 }
895 
896 SDValue SystemZDAGToDAGISel::convertTo(SDLoc DL, EVT VT, SDValue N) const {
897  if (N.getValueType() == MVT::i32 && VT == MVT::i64)
898  return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
899  DL, VT, getUNDEF(DL, MVT::i64), N);
900  if (N.getValueType() == MVT::i64 && VT == MVT::i32)
901  return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
902  assert(N.getValueType() == VT && "Unexpected value types");
903  return N;
904 }
905 
906 SDNode *SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
907  SDLoc DL(N);
908  EVT VT = N->getValueType(0);
909  if (!VT.isInteger() || VT.getSizeInBits() > 64)
910  return nullptr;
911  RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
912  unsigned Count = 0;
913  while (expandRxSBG(RISBG))
914  if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND)
915  Count += 1;
916  if (Count == 0)
917  return nullptr;
918  if (Count == 1) {
919  // Prefer to use normal shift instructions over RISBG, since they can handle
920  // all cases and are sometimes shorter.
921  if (N->getOpcode() != ISD::AND)
922  return nullptr;
923 
924  // Prefer register extensions like LLC over RISBG. Also prefer to start
925  // out with normal ANDs if one instruction would be enough. We can convert
926  // these ANDs into an RISBG later if a three-address instruction is useful.
927  if (VT == MVT::i32 ||
928  RISBG.Mask == 0xff ||
929  RISBG.Mask == 0xffff ||
930  SystemZ::isImmLF(~RISBG.Mask) ||
931  SystemZ::isImmHF(~RISBG.Mask)) {
932  // Force the new mask into the DAG, since it may include known-one bits.
933  auto *MaskN = cast<ConstantSDNode>(N->getOperand(1).getNode());
934  if (MaskN->getZExtValue() != RISBG.Mask) {
935  SDValue NewMask = CurDAG->getConstant(RISBG.Mask, DL, VT);
936  N = CurDAG->UpdateNodeOperands(N, N->getOperand(0), NewMask);
937  return SelectCode(N);
938  }
939  return nullptr;
940  }
941  }
942 
943  unsigned Opcode = SystemZ::RISBG;
944  // Prefer RISBGN if available, since it does not clobber CC.
945  if (Subtarget->hasMiscellaneousExtensions())
946  Opcode = SystemZ::RISBGN;
947  EVT OpcodeVT = MVT::i64;
948  if (VT == MVT::i32 && Subtarget->hasHighWord()) {
949  Opcode = SystemZ::RISBMux;
950  OpcodeVT = MVT::i32;
951  RISBG.Start &= 31;
952  RISBG.End &= 31;
953  }
954  SDValue Ops[5] = {
955  getUNDEF(DL, OpcodeVT),
956  convertTo(DL, OpcodeVT, RISBG.Input),
957  CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
958  CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
959  CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
960  };
961  N = CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops);
962  return convertTo(DL, VT, SDValue(N, 0)).getNode();
963 }
964 
965 SDNode *SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
966  SDLoc DL(N);
967  EVT VT = N->getValueType(0);
968  if (!VT.isInteger() || VT.getSizeInBits() > 64)
969  return nullptr;
970  // Try treating each operand of N as the second operand of the RxSBG
971  // and see which goes deepest.
972  RxSBGOperands RxSBG[] = {
973  RxSBGOperands(Opcode, N->getOperand(0)),
974  RxSBGOperands(Opcode, N->getOperand(1))
975  };
976  unsigned Count[] = { 0, 0 };
977  for (unsigned I = 0; I < 2; ++I)
978  while (expandRxSBG(RxSBG[I]))
979  if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND)
980  Count[I] += 1;
981 
982  // Do nothing if neither operand is suitable.
983  if (Count[0] == 0 && Count[1] == 0)
984  return nullptr;
985 
986  // Pick the deepest second operand.
987  unsigned I = Count[0] > Count[1] ? 0 : 1;
988  SDValue Op0 = N->getOperand(I ^ 1);
989 
990  // Prefer IC for character insertions from memory.
991  if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
992  if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
993  if (Load->getMemoryVT() == MVT::i8)
994  return nullptr;
995 
996  // See whether we can avoid an AND in the first operand by converting
997  // ROSBG to RISBG.
998  if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
999  Opcode = SystemZ::RISBG;
1000  // Prefer RISBGN if available, since it does not clobber CC.
1001  if (Subtarget->hasMiscellaneousExtensions())
1002  Opcode = SystemZ::RISBGN;
1003  }
1004 
1005  SDValue Ops[5] = {
1006  convertTo(DL, MVT::i64, Op0),
1007  convertTo(DL, MVT::i64, RxSBG[I].Input),
1008  CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1009  CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1010  CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1011  };
1012  N = CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops);
1013  return convertTo(DL, VT, SDValue(N, 0)).getNode();
1014 }
1015 
1016 SDNode *SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1017  SDValue Op0, uint64_t UpperVal,
1018  uint64_t LowerVal) {
1019  EVT VT = Node->getValueType(0);
1020  SDLoc DL(Node);
1021  SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1022  if (Op0.getNode())
1023  Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1024  Upper = SDValue(Select(Upper.getNode()), 0);
1025 
1026  SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1027  SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1028  return Or.getNode();
1029 }
1030 
1031 SDNode *SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1032  SDValue ElemV = N->getOperand(2);
1033  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1034  if (!ElemN)
1035  return 0;
1036 
1037  unsigned Elem = ElemN->getZExtValue();
1038  EVT VT = N->getValueType(0);
1039  if (Elem >= VT.getVectorNumElements())
1040  return 0;
1041 
1042  auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1043  if (!Load || !Load->hasOneUse())
1044  return 0;
1045  if (Load->getMemoryVT().getSizeInBits() !=
1046  Load->getValueType(0).getSizeInBits())
1047  return 0;
1048 
1049  SDValue Base, Disp, Index;
1050  if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1052  return 0;
1053 
1054  SDLoc DL(Load);
1055  SDValue Ops[] = {
1056  N->getOperand(0), Base, Disp, Index,
1057  CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1058  };
1059  SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1060  ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1061  return Res;
1062 }
1063 
1064 SDNode *SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1065  SDValue Value = Store->getValue();
1066  if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1067  return 0;
1068  if (Store->getMemoryVT().getSizeInBits() !=
1069  Value.getValueType().getSizeInBits())
1070  return 0;
1071 
1072  SDValue ElemV = Value.getOperand(1);
1073  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1074  if (!ElemN)
1075  return 0;
1076 
1077  SDValue Vec = Value.getOperand(0);
1078  EVT VT = Vec.getValueType();
1079  unsigned Elem = ElemN->getZExtValue();
1080  if (Elem >= VT.getVectorNumElements())
1081  return 0;
1082 
1083  SDValue Base, Disp, Index;
1084  if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1086  return 0;
1087 
1088  SDLoc DL(Store);
1089  SDValue Ops[] = {
1090  Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1091  Store->getChain()
1092  };
1093  return CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
1094 }
1095 
1096 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1097  LoadSDNode *Load) const {
1098  // Check that the two memory operands have the same size.
1099  if (Load->getMemoryVT() != Store->getMemoryVT())
1100  return false;
1101 
1102  // Volatility stops an access from being decomposed.
1103  if (Load->isVolatile() || Store->isVolatile())
1104  return false;
1105 
1106  // There's no chance of overlap if the load is invariant.
1107  if (Load->isInvariant())
1108  return true;
1109 
1110  // Otherwise we need to check whether there's an alias.
1111  const Value *V1 = Load->getMemOperand()->getValue();
1112  const Value *V2 = Store->getMemOperand()->getValue();
1113  if (!V1 || !V2)
1114  return false;
1115 
1116  // Reject equality.
1117  uint64_t Size = Load->getMemoryVT().getStoreSize();
1118  int64_t End1 = Load->getSrcValueOffset() + Size;
1119  int64_t End2 = Store->getSrcValueOffset() + Size;
1120  if (V1 == V2 && End1 == End2)
1121  return false;
1122 
1123  return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()),
1124  MemoryLocation(V2, End2, Store->getAAInfo()));
1125 }
1126 
1127 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1128  auto *Store = cast<StoreSDNode>(N);
1129  auto *Load = cast<LoadSDNode>(Store->getValue());
1130 
1131  // Prefer not to use MVC if either address can use ... RELATIVE LONG
1132  // instructions.
1133  uint64_t Size = Load->getMemoryVT().getStoreSize();
1134  if (Size > 1 && Size <= 8) {
1135  // Prefer LHRL, LRL and LGRL.
1136  if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1137  return false;
1138  // Prefer STHRL, STRL and STGRL.
1139  if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1140  return false;
1141  }
1142 
1143  return canUseBlockOperation(Store, Load);
1144 }
1145 
1146 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1147  unsigned I) const {
1148  auto *StoreA = cast<StoreSDNode>(N);
1149  auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1150  auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1151  return !LoadA->isVolatile() && canUseBlockOperation(StoreA, LoadB);
1152 }
1153 
1155  // Dump information about the Node being selected
1156  DEBUG(errs() << "Selecting: "; Node->dump(CurDAG); errs() << "\n");
1157 
1158  // If we have a custom node, we already have selected!
1159  if (Node->isMachineOpcode()) {
1160  DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1161  Node->setNodeId(-1);
1162  return nullptr;
1163  }
1164 
1165  unsigned Opcode = Node->getOpcode();
1166  SDNode *ResNode = nullptr;
1167  switch (Opcode) {
1168  case ISD::OR:
1169  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1170  ResNode = tryRxSBG(Node, SystemZ::ROSBG);
1171  goto or_xor;
1172 
1173  case ISD::XOR:
1174  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1175  ResNode = tryRxSBG(Node, SystemZ::RXSBG);
1176  // Fall through.
1177  or_xor:
1178  // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1179  // split the operation into two.
1180  if (!ResNode && Node->getValueType(0) == MVT::i64)
1181  if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1182  uint64_t Val = Op1->getZExtValue();
1183  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val))
1184  Node = splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1185  Val - uint32_t(Val), uint32_t(Val));
1186  }
1187  break;
1188 
1189  case ISD::AND:
1190  if (Node->getOperand(1).getOpcode() != ISD::Constant)
1191  ResNode = tryRxSBG(Node, SystemZ::RNSBG);
1192  // Fall through.
1193  case ISD::ROTL:
1194  case ISD::SHL:
1195  case ISD::SRL:
1196  case ISD::ZERO_EXTEND:
1197  if (!ResNode)
1198  ResNode = tryRISBGZero(Node);
1199  break;
1200 
1201  case ISD::Constant:
1202  // If this is a 64-bit constant that is out of the range of LLILF,
1203  // LLIHF and LGFI, split it into two 32-bit pieces.
1204  if (Node->getValueType(0) == MVT::i64) {
1205  uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1206  if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val))
1207  Node = splitLargeImmediate(ISD::OR, Node, SDValue(),
1208  Val - uint32_t(Val), uint32_t(Val));
1209  }
1210  break;
1211 
1213  SDValue Op0 = Node->getOperand(0);
1214  SDValue Op1 = Node->getOperand(1);
1215  // Prefer to put any load first, so that it can be matched as a
1216  // conditional load.
1217  if (Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) {
1218  SDValue CCValid = Node->getOperand(2);
1219  SDValue CCMask = Node->getOperand(3);
1220  uint64_t ConstCCValid =
1221  cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1222  uint64_t ConstCCMask =
1223  cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1224  // Invert the condition.
1225  CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node),
1226  CCMask.getValueType());
1227  SDValue Op4 = Node->getOperand(4);
1228  Node = CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1229  }
1230  break;
1231  }
1232 
1233  case ISD::INSERT_VECTOR_ELT: {
1234  EVT VT = Node->getValueType(0);
1235  unsigned ElemBitSize = VT.getVectorElementType().getSizeInBits();
1236  if (ElemBitSize == 32)
1237  ResNode = tryGather(Node, SystemZ::VGEF);
1238  else if (ElemBitSize == 64)
1239  ResNode = tryGather(Node, SystemZ::VGEG);
1240  break;
1241  }
1242 
1243  case ISD::STORE: {
1244  auto *Store = cast<StoreSDNode>(Node);
1245  unsigned ElemBitSize = Store->getValue().getValueType().getSizeInBits();
1246  if (ElemBitSize == 32)
1247  ResNode = tryScatter(Store, SystemZ::VSCEF);
1248  else if (ElemBitSize == 64)
1249  ResNode = tryScatter(Store, SystemZ::VSCEG);
1250  break;
1251  }
1252  }
1253 
1254  // Select the default instruction
1255  if (!ResNode)
1256  ResNode = SelectCode(Node);
1257 
1258  DEBUG(errs() << "=> ";
1259  if (ResNode == nullptr || ResNode == Node)
1260  Node->dump(CurDAG);
1261  else
1262  ResNode->dump(CurDAG);
1263  errs() << "\n";
1264  );
1265  return ResNode;
1266 }
1267 
1268 bool SystemZDAGToDAGISel::
1269 SelectInlineAsmMemoryOperand(const SDValue &Op,
1270  unsigned ConstraintID,
1271  std::vector<SDValue> &OutOps) {
1272  switch(ConstraintID) {
1273  default:
1274  llvm_unreachable("Unexpected asm memory constraint");
1281  // Accept addresses with short displacements, which are compatible
1282  // with Q, R, S and T. But keep the index operand for future expansion.
1283  SDValue Base, Disp, Index;
1284  if (selectBDXAddr(SystemZAddressingMode::FormBD,
1285  SystemZAddressingMode::Disp12Only,
1286  Op, Base, Disp, Index)) {
1287  OutOps.push_back(Base);
1288  OutOps.push_back(Disp);
1289  OutOps.push_back(Index);
1290  return false;
1291  }
1292  break;
1293  }
1294  return true;
1295 }
bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:276
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:130
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1327
void dump() const
Dump this node, for debugging.
bool hasOneUse() const
Return true if there is exactly one use of this node.
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:125
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
const SDValue & getOperand(unsigned Num) const
void setNodeId(int Id)
Set unique node id.
static std::error_code getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
int64_t getSrcValueOffset() const
static uint64_t allOnes(unsigned int Count)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const HexagonInstrInfo * TII
Shift and rotation operations.
Definition: ISDOpcodes.h:332
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:110
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:216
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:191
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
const SDValue & getBasePtr() const
EVT getMemoryVT() const
Return the type of the in-memory value.
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
This class is used to represent ISD::STORE nodes.
SDNode * getNode() const
get the SDNode which holds the desired result
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
Definition: TargetOpcodes.h:52
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:245
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
const SDValue & getOperand(unsigned i) const
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:267
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1895
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
bool isPCREL(unsigned Opcode)
bool isVolatile() const
const SDValue & getValue() const
bool isRxSBGMask(uint64_t Mask, unsigned BitSize, unsigned &Start, unsigned &End) const
EVT - Extended Value Type.
Definition: ValueTypes.h:31
Representation for a specific memory location.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:273
bool isInvariant() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:179
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Target - Wrapper for Target specific information.
Class for arbitrary precision integers.
Definition: APInt.h:73
const Value * getValue() const
getValue - Return the base address of the memory access.
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:383
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:386
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:321
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:401
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:518
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
bool isInt< 16 >(int64_t x)
Definition: MathExtras.h:272
int getNodeId() const
Return the unique node id.
EVT getValueType() const
Return the ValueType of the referenced return value.
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
LLVM Value Representation.
Definition: Value.h:69
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
#define DEBUG(X)
Definition: Debug.h:92
Conversion operators.
Definition: ISDOpcodes.h:380
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:389
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
EVT changeVectorElementTypeToInteger() const
changeVectorElementTypeToInteger - Return a vector with the same number of elements as this vector...
Definition: ValueTypes.h:80
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
uint64_t getZExtValue() const
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:225
This class is used to represent ISD::LOAD nodes.