LLVM 18.0.0git
SystemZISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the SystemZ target.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SystemZISelLowering.h"
17#include "llvm/Support/Debug.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "systemz-isel"
24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
25
26namespace {
27// Used to build addressing modes.
28struct SystemZAddressingMode {
29 // The shape of the address.
30 enum AddrForm {
31 // base+displacement
32 FormBD,
33
34 // base+displacement+index for load and store operands
35 FormBDXNormal,
36
37 // base+displacement+index for load address operands
38 FormBDXLA,
39
40 // base+displacement+index+ADJDYNALLOC
41 FormBDXDynAlloc
42 };
43 AddrForm Form;
44
45 // The type of displacement. The enum names here correspond directly
46 // to the definitions in SystemZOperand.td. We could split them into
47 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
48 enum DispRange {
49 Disp12Only,
50 Disp12Pair,
51 Disp20Only,
52 Disp20Only128,
53 Disp20Pair
54 };
55 DispRange DR;
56
57 // The parts of the address. The address is equivalent to:
58 //
59 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
61 int64_t Disp;
63 bool IncludesDynAlloc;
64
65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
67
68 // True if the address can have an index register.
69 bool hasIndexField() { return Form != FormBD; }
70
71 // True if the address can (and must) include ADJDYNALLOC.
72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
74 void dump(const llvm::SelectionDAG *DAG) {
75 errs() << "SystemZAddressingMode " << this << '\n';
76
77 errs() << " Base ";
78 if (Base.getNode())
79 Base.getNode()->dump(DAG);
80 else
81 errs() << "null\n";
82
83 if (hasIndexField()) {
84 errs() << " Index ";
85 if (Index.getNode())
86 Index.getNode()->dump(DAG);
87 else
88 errs() << "null\n";
89 }
90
91 errs() << " Disp " << Disp;
92 if (IncludesDynAlloc)
93 errs() << " + ADJDYNALLOC";
94 errs() << '\n';
95 }
96};
97
98// Return a mask with Count low bits set.
99static uint64_t allOnes(unsigned int Count) {
100 assert(Count <= 64);
101 if (Count > 63)
102 return UINT64_MAX;
103 return (uint64_t(1) << Count) - 1;
104}
105
106// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107// given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108// Rotate (I5). The combined operand value is effectively:
109//
110// (or (rotl Input, Rotate), ~Mask)
111//
112// for RNSBG and:
113//
114// (and (rotl Input, Rotate), Mask)
115//
116// otherwise. The output value has BitSize bits, although Input may be
117// narrower (in which case the upper bits are don't care), or wider (in which
118// case the result will be truncated as part of the operation).
119struct RxSBGOperands {
120 RxSBGOperands(unsigned Op, SDValue N)
121 : Opcode(Op), BitSize(N.getValueSizeInBits()),
122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123 Rotate(0) {}
124
125 unsigned Opcode;
126 unsigned BitSize;
127 uint64_t Mask;
128 SDValue Input;
129 unsigned Start;
130 unsigned End;
131 unsigned Rotate;
132};
133
134class SystemZDAGToDAGISel : public SelectionDAGISel {
135 const SystemZSubtarget *Subtarget;
136
137 // Used by SystemZOperands.td to create integer constants.
138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140 }
141
142 const SystemZTargetMachine &getTargetMachine() const {
143 return static_cast<const SystemZTargetMachine &>(TM);
144 }
145
146 const SystemZInstrInfo *getInstrInfo() const {
147 return Subtarget->getInstrInfo();
148 }
149
150 // Try to fold more of the base or index of AM into AM, where IsBase
151 // selects between the base and index.
152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154 // Try to describe N in AM, returning true on success.
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157 // Extract individual target operands from matched address AM.
158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159 SDValue &Base, SDValue &Disp) const;
160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163 // Try to match Addr as a FormBD address with displacement type DR.
164 // Return true on success, storing the base and displacement in
165 // Base and Disp respectively.
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
168
169 // Try to match Addr as a FormBDX address with displacement type DR.
170 // Return true on success and if the result had no index. Store the
171 // base and displacement in Base and Disp respectively.
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
174
175 // Try to match Addr as a FormBDX* address of form Form with
176 // displacement type DR. Return true on success, storing the base,
177 // displacement and index in Base, Disp and Index respectively.
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182 // PC-relative address matching routines used by SystemZOperands.td.
183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184 if (SystemZISD::isPCREL(Addr.getOpcode())) {
185 Target = Addr.getOperand(0);
186 return true;
187 }
188 return false;
189 }
190
191 // BD matching routines used by SystemZOperands.td.
192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194 }
195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197 }
198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200 }
201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203 }
204
205 // MVI matching routines used by SystemZOperands.td.
206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208 }
209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211 }
212
213 // BDX matching routines used by SystemZOperands.td.
214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
218 Addr, Base, Disp, Index);
219 }
220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
224 Addr, Base, Disp, Index);
225 }
226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
230 Addr, Base, Disp, Index);
231 }
232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
236 Addr, Base, Disp, Index);
237 }
238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
242 Addr, Base, Disp, Index);
243 }
244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
248 Addr, Base, Disp, Index);
249 }
250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
254 Addr, Base, Disp, Index);
255 }
256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
260 Addr, Base, Disp, Index);
261 }
262
263 // Try to match Addr as an address with a base, 12-bit displacement
264 // and index, where the index is element Elem of a vector.
265 // Return true on success, storing the base, displacement and vector
266 // in Base, Disp and Index respectively.
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
269
270 // Check whether (or Op (and X InsertMask)) is effectively an insertion
271 // of X into bits InsertMask of some Y != Op. Return true if so and
272 // set Op to that Y.
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276 // Return true on success.
277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279 // Try to fold some of RxSBG.Input into other fields of RxSBG.
280 // Return true on success.
281 bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283 // Return an undefined value of type VT.
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286 // Convert N to VT, if it isn't already.
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289 // Try to implement AND or shift node N using RISBG with the zero flag set.
290 // Return the selected node on success, otherwise return null.
291 bool tryRISBGZero(SDNode *N);
292
293 // Try to use RISBG or Opcode to implement OR or XOR node N.
294 // Return the selected node on success, otherwise return null.
295 bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297 // If Op0 is null, then Node is a constant that can be loaded using:
298 //
299 // (Opcode UpperVal LowerVal)
300 //
301 // If Op0 is nonnull, then Node can be implemented using:
302 //
303 // (Opcode (Opcode Op0 UpperVal) LowerVal)
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305 uint64_t UpperVal, uint64_t LowerVal);
306
307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
308 SDNode *Node);
309
310 // Try to use gather instruction Opcode to implement vector insertion N.
311 bool tryGather(SDNode *N, unsigned Opcode);
312
313 // Try to use scatter instruction Opcode to implement store Store.
314 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
315
316 // Change a chain of {load; op; store} of the same value into a simple op
317 // through memory of that value, if the uses of the modified value and its
318 // address are suitable.
319 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
320
321 // Return true if Load and Store are loads and stores of the same size
322 // and are guaranteed not to overlap. Such operations can be implemented
323 // using block (SS-format) instructions.
324 //
325 // Partial overlap would lead to incorrect code, since the block operations
326 // are logically bytewise, even though they have a fast path for the
327 // non-overlapping case. We also need to avoid full overlap (i.e. two
328 // addresses that might be equal at run time) because although that case
329 // would be handled correctly, it might be implemented by millicode.
330 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
331
332 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
333 // from Y to X.
334 bool storeLoadCanUseMVC(SDNode *N) const;
335
336 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
337 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
338 // to X.
339 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
340
341 // Return true if N (a load or a store) fullfills the alignment
342 // requirements for a PC-relative access.
343 bool storeLoadIsAligned(SDNode *N) const;
344
345 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
346 SDValue expandSelectBoolean(SDNode *Node);
347
348public:
349 static char ID;
350
351 SystemZDAGToDAGISel() = delete;
352
353 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
354 : SelectionDAGISel(ID, TM, OptLevel) {}
355
356 bool runOnMachineFunction(MachineFunction &MF) override {
357 const Function &F = MF.getFunction();
358 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
359 if (F.hasFnAttribute("mnop-mcount"))
360 report_fatal_error("mnop-mcount only supported with fentry-call");
361 if (F.hasFnAttribute("mrecord-mcount"))
362 report_fatal_error("mrecord-mcount only supported with fentry-call");
363 }
364
365 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
367 }
368
369 // Override SelectionDAGISel.
370 void Select(SDNode *Node) override;
372 InlineAsm::ConstraintCode ConstraintID,
373 std::vector<SDValue> &OutOps) override;
374 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
375 void PreprocessISelDAG() override;
376
377 // Include the pieces autogenerated from the target description.
378 #include "SystemZGenDAGISel.inc"
379};
380} // end anonymous namespace
381
382char SystemZDAGToDAGISel::ID = 0;
383
384INITIALIZE_PASS(SystemZDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false)
385
387 CodeGenOptLevel OptLevel) {
388 return new SystemZDAGToDAGISel(TM, OptLevel);
389}
390
391// Return true if Val should be selected as a displacement for an address
392// with range DR. Here we're interested in the range of both the instruction
393// described by DR and of any pairing instruction.
394static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
395 switch (DR) {
396 case SystemZAddressingMode::Disp12Only:
397 return isUInt<12>(Val);
398
399 case SystemZAddressingMode::Disp12Pair:
400 case SystemZAddressingMode::Disp20Only:
401 case SystemZAddressingMode::Disp20Pair:
402 return isInt<20>(Val);
403
404 case SystemZAddressingMode::Disp20Only128:
405 return isInt<20>(Val) && isInt<20>(Val + 8);
406 }
407 llvm_unreachable("Unhandled displacement range");
408}
409
410// Change the base or index in AM to Value, where IsBase selects
411// between the base and index.
412static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
413 SDValue Value) {
414 if (IsBase)
415 AM.Base = Value;
416 else
417 AM.Index = Value;
418}
419
420// The base or index of AM is equivalent to Value + ADJDYNALLOC,
421// where IsBase selects between the base and index. Try to fold the
422// ADJDYNALLOC into AM.
423static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
424 SDValue Value) {
425 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
426 changeComponent(AM, IsBase, Value);
427 AM.IncludesDynAlloc = true;
428 return true;
429 }
430 return false;
431}
432
433// The base of AM is equivalent to Base + Index. Try to use Index as
434// the index register.
435static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
436 SDValue Index) {
437 if (AM.hasIndexField() && !AM.Index.getNode()) {
438 AM.Base = Base;
439 AM.Index = Index;
440 return true;
441 }
442 return false;
443}
444
445// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
446// between the base and index. Try to fold Op1 into AM's displacement.
447static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
448 SDValue Op0, uint64_t Op1) {
449 // First try adjusting the displacement.
450 int64_t TestDisp = AM.Disp + Op1;
451 if (selectDisp(AM.DR, TestDisp)) {
452 changeComponent(AM, IsBase, Op0);
453 AM.Disp = TestDisp;
454 return true;
455 }
456
457 // We could consider forcing the displacement into a register and
458 // using it as an index, but it would need to be carefully tuned.
459 return false;
460}
461
462bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
463 bool IsBase) const {
464 SDValue N = IsBase ? AM.Base : AM.Index;
465 unsigned Opcode = N.getOpcode();
466 if (Opcode == ISD::TRUNCATE) {
467 N = N.getOperand(0);
468 Opcode = N.getOpcode();
469 }
470 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
471 SDValue Op0 = N.getOperand(0);
472 SDValue Op1 = N.getOperand(1);
473
474 unsigned Op0Code = Op0->getOpcode();
475 unsigned Op1Code = Op1->getOpcode();
476
477 if (Op0Code == SystemZISD::ADJDYNALLOC)
478 return expandAdjDynAlloc(AM, IsBase, Op1);
479 if (Op1Code == SystemZISD::ADJDYNALLOC)
480 return expandAdjDynAlloc(AM, IsBase, Op0);
481
482 if (Op0Code == ISD::Constant)
483 return expandDisp(AM, IsBase, Op1,
484 cast<ConstantSDNode>(Op0)->getSExtValue());
485 if (Op1Code == ISD::Constant)
486 return expandDisp(AM, IsBase, Op0,
487 cast<ConstantSDNode>(Op1)->getSExtValue());
488
489 if (IsBase && expandIndex(AM, Op0, Op1))
490 return true;
491 }
493 SDValue Full = N.getOperand(0);
494 SDValue Base = N.getOperand(1);
495 SDValue Anchor = Base.getOperand(0);
496 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
497 cast<GlobalAddressSDNode>(Anchor)->getOffset());
498 return expandDisp(AM, IsBase, Base, Offset);
499 }
500 return false;
501}
502
503// Return true if an instruction with displacement range DR should be
504// used for displacement value Val. selectDisp(DR, Val) must already hold.
505static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
506 assert(selectDisp(DR, Val) && "Invalid displacement");
507 switch (DR) {
508 case SystemZAddressingMode::Disp12Only:
509 case SystemZAddressingMode::Disp20Only:
510 case SystemZAddressingMode::Disp20Only128:
511 return true;
512
513 case SystemZAddressingMode::Disp12Pair:
514 // Use the other instruction if the displacement is too large.
515 return isUInt<12>(Val);
516
517 case SystemZAddressingMode::Disp20Pair:
518 // Use the other instruction if the displacement is small enough.
519 return !isUInt<12>(Val);
520 }
521 llvm_unreachable("Unhandled displacement range");
522}
523
524// Return true if Base + Disp + Index should be performed by LA(Y).
525static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
526 // Don't use LA(Y) for constants.
527 if (!Base)
528 return false;
529
530 // Always use LA(Y) for frame addresses, since we know that the destination
531 // register is almost always (perhaps always) going to be different from
532 // the frame register.
533 if (Base->getOpcode() == ISD::FrameIndex)
534 return true;
535
536 if (Disp) {
537 // Always use LA(Y) if there is a base, displacement and index.
538 if (Index)
539 return true;
540
541 // Always use LA if the displacement is small enough. It should always
542 // be no worse than AGHI (and better if it avoids a move).
543 if (isUInt<12>(Disp))
544 return true;
545
546 // For similar reasons, always use LAY if the constant is too big for AGHI.
547 // LAY should be no worse than AGFI.
548 if (!isInt<16>(Disp))
549 return true;
550 } else {
551 // Don't use LA for plain registers.
552 if (!Index)
553 return false;
554
555 // Don't use LA for plain addition if the index operand is only used
556 // once. It should be a natural two-operand addition in that case.
557 if (Index->hasOneUse())
558 return false;
559
560 // Prefer addition if the second operation is sign-extended, in the
561 // hope of using AGF.
562 unsigned IndexOpcode = Index->getOpcode();
563 if (IndexOpcode == ISD::SIGN_EXTEND ||
564 IndexOpcode == ISD::SIGN_EXTEND_INREG)
565 return false;
566 }
567
568 // Don't use LA for two-operand addition if either operand is only
569 // used once. The addition instructions are better in that case.
570 if (Base->hasOneUse())
571 return false;
572
573 return true;
574}
575
576// Return true if Addr is suitable for AM, updating AM if so.
577bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
578 SystemZAddressingMode &AM) const {
579 // Start out assuming that the address will need to be loaded separately,
580 // then try to extend it as much as we can.
581 AM.Base = Addr;
582
583 // First try treating the address as a constant.
584 if (Addr.getOpcode() == ISD::Constant &&
585 expandDisp(AM, true, SDValue(),
586 cast<ConstantSDNode>(Addr)->getSExtValue()))
587 ;
588 // Also see if it's a bare ADJDYNALLOC.
589 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
590 expandAdjDynAlloc(AM, true, SDValue()))
591 ;
592 else
593 // Otherwise try expanding each component.
594 while (expandAddress(AM, true) ||
595 (AM.Index.getNode() && expandAddress(AM, false)))
596 continue;
597
598 // Reject cases where it isn't profitable to use LA(Y).
599 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
600 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
601 return false;
602
603 // Reject cases where the other instruction in a pair should be used.
604 if (!isValidDisp(AM.DR, AM.Disp))
605 return false;
606
607 // Make sure that ADJDYNALLOC is included where necessary.
608 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
609 return false;
610
611 LLVM_DEBUG(AM.dump(CurDAG));
612 return true;
613}
614
615// Insert a node into the DAG at least before Pos. This will reposition
616// the node as needed, and will assign it a node ID that is <= Pos's ID.
617// Note that this does *not* preserve the uniqueness of node IDs!
618// The selection DAG must no longer depend on their uniqueness when this
619// function is used.
620static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
621 if (N->getNodeId() == -1 ||
624 DAG->RepositionNode(Pos->getIterator(), N.getNode());
625 // Mark Node as invalid for pruning as after this it may be a successor to a
626 // selected node but otherwise be in the same position of Pos.
627 // Conservatively mark it with the same -abs(Id) to assure node id
628 // invariant is preserved.
629 N->setNodeId(Pos->getNodeId());
631 }
632}
633
634void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
635 EVT VT, SDValue &Base,
636 SDValue &Disp) const {
637 Base = AM.Base;
638 if (!Base.getNode())
639 // Register 0 means "no base". This is mostly useful for shifts.
640 Base = CurDAG->getRegister(0, VT);
641 else if (Base.getOpcode() == ISD::FrameIndex) {
642 // Lower a FrameIndex to a TargetFrameIndex.
643 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
644 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
645 } else if (Base.getValueType() != VT) {
646 // Truncate values from i64 to i32, for shifts.
647 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
648 "Unexpected truncation");
649 SDLoc DL(Base);
650 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
651 insertDAGNode(CurDAG, Base.getNode(), Trunc);
652 Base = Trunc;
653 }
654
655 // Lower the displacement to a TargetConstant.
656 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
657}
658
659void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
660 EVT VT, SDValue &Base,
661 SDValue &Disp,
662 SDValue &Index) const {
663 getAddressOperands(AM, VT, Base, Disp);
664
665 Index = AM.Index;
666 if (!Index.getNode())
667 // Register 0 means "no index".
668 Index = CurDAG->getRegister(0, VT);
669}
670
671bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
673 SDValue &Disp) const {
674 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
675 if (!selectAddress(Addr, AM))
676 return false;
677
678 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
679 return true;
680}
681
682bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
684 SDValue &Disp) const {
685 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
686 if (!selectAddress(Addr, AM) || AM.Index.getNode())
687 return false;
688
689 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
690 return true;
691}
692
693bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
694 SystemZAddressingMode::DispRange DR,
696 SDValue &Disp, SDValue &Index) const {
697 SystemZAddressingMode AM(Form, DR);
698 if (!selectAddress(Addr, AM))
699 return false;
700
701 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
702 return true;
703}
704
705bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
706 SDValue &Base,
707 SDValue &Disp,
708 SDValue &Index) const {
709 SDValue Regs[2];
710 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
711 Regs[0].getNode() && Regs[1].getNode()) {
712 for (unsigned int I = 0; I < 2; ++I) {
713 Base = Regs[I];
714 Index = Regs[1 - I];
715 // We can't tell here whether the index vector has the right type
716 // for the access; the caller needs to do that instead.
717 if (Index.getOpcode() == ISD::ZERO_EXTEND)
719 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
720 Index.getOperand(1) == Elem) {
722 return true;
723 }
724 }
725 }
726 return false;
727}
728
729bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
730 uint64_t InsertMask) const {
731 // We're only interested in cases where the insertion is into some operand
732 // of Op, rather than into Op itself. The only useful case is an AND.
733 if (Op.getOpcode() != ISD::AND)
734 return false;
735
736 // We need a constant mask.
737 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
738 if (!MaskNode)
739 return false;
740
741 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
742 uint64_t AndMask = MaskNode->getZExtValue();
743 if (InsertMask & AndMask)
744 return false;
745
746 // It's only an insertion if all bits are covered or are known to be zero.
747 // The inner check covers all cases but is more expensive.
748 uint64_t Used = allOnes(Op.getValueSizeInBits());
749 if (Used != (AndMask | InsertMask)) {
750 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
751 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
752 return false;
753 }
754
755 Op = Op.getOperand(0);
756 return true;
757}
758
759bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
760 uint64_t Mask) const {
761 const SystemZInstrInfo *TII = getInstrInfo();
762 if (RxSBG.Rotate != 0)
763 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
764 Mask &= RxSBG.Mask;
765 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
766 RxSBG.Mask = Mask;
767 return true;
768 }
769 return false;
770}
771
772// Return true if any bits of (RxSBG.Input & Mask) are significant.
773static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
774 // Rotate the mask in the same way as RxSBG.Input is rotated.
775 if (RxSBG.Rotate != 0)
776 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
777 return (Mask & RxSBG.Mask) != 0;
778}
779
780bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
781 SDValue N = RxSBG.Input;
782 unsigned Opcode = N.getOpcode();
783 switch (Opcode) {
784 case ISD::TRUNCATE: {
785 if (RxSBG.Opcode == SystemZ::RNSBG)
786 return false;
787 uint64_t BitSize = N.getValueSizeInBits();
788 uint64_t Mask = allOnes(BitSize);
789 if (!refineRxSBGMask(RxSBG, Mask))
790 return false;
791 RxSBG.Input = N.getOperand(0);
792 return true;
793 }
794 case ISD::AND: {
795 if (RxSBG.Opcode == SystemZ::RNSBG)
796 return false;
797
798 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
799 if (!MaskNode)
800 return false;
801
802 SDValue Input = N.getOperand(0);
803 uint64_t Mask = MaskNode->getZExtValue();
804 if (!refineRxSBGMask(RxSBG, Mask)) {
805 // If some bits of Input are already known zeros, those bits will have
806 // been removed from the mask. See if adding them back in makes the
807 // mask suitable.
808 KnownBits Known = CurDAG->computeKnownBits(Input);
809 Mask |= Known.Zero.getZExtValue();
810 if (!refineRxSBGMask(RxSBG, Mask))
811 return false;
812 }
813 RxSBG.Input = Input;
814 return true;
815 }
816
817 case ISD::OR: {
818 if (RxSBG.Opcode != SystemZ::RNSBG)
819 return false;
820
821 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
822 if (!MaskNode)
823 return false;
824
825 SDValue Input = N.getOperand(0);
826 uint64_t Mask = ~MaskNode->getZExtValue();
827 if (!refineRxSBGMask(RxSBG, Mask)) {
828 // If some bits of Input are already known ones, those bits will have
829 // been removed from the mask. See if adding them back in makes the
830 // mask suitable.
831 KnownBits Known = CurDAG->computeKnownBits(Input);
832 Mask &= ~Known.One.getZExtValue();
833 if (!refineRxSBGMask(RxSBG, Mask))
834 return false;
835 }
836 RxSBG.Input = Input;
837 return true;
838 }
839
840 case ISD::ROTL: {
841 // Any 64-bit rotate left can be merged into the RxSBG.
842 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
843 return false;
844 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
845 if (!CountNode)
846 return false;
847
848 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
849 RxSBG.Input = N.getOperand(0);
850 return true;
851 }
852
853 case ISD::ANY_EXTEND:
854 // Bits above the extended operand are don't-care.
855 RxSBG.Input = N.getOperand(0);
856 return true;
857
858 case ISD::ZERO_EXTEND:
859 if (RxSBG.Opcode != SystemZ::RNSBG) {
860 // Restrict the mask to the extended operand.
861 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
862 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
863 return false;
864
865 RxSBG.Input = N.getOperand(0);
866 return true;
867 }
868 [[fallthrough]];
869
870 case ISD::SIGN_EXTEND: {
871 // Check that the extension bits are don't-care (i.e. are masked out
872 // by the final mask).
873 unsigned BitSize = N.getValueSizeInBits();
874 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
875 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
876 // In the case where only the sign bit is active, increase Rotate with
877 // the extension width.
878 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
879 RxSBG.Rotate += (BitSize - InnerBitSize);
880 else
881 return false;
882 }
883
884 RxSBG.Input = N.getOperand(0);
885 return true;
886 }
887
888 case ISD::SHL: {
889 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
890 if (!CountNode)
891 return false;
892
893 uint64_t Count = CountNode->getZExtValue();
894 unsigned BitSize = N.getValueSizeInBits();
895 if (Count < 1 || Count >= BitSize)
896 return false;
897
898 if (RxSBG.Opcode == SystemZ::RNSBG) {
899 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
900 // count bits from RxSBG.Input are ignored.
901 if (maskMatters(RxSBG, allOnes(Count)))
902 return false;
903 } else {
904 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
905 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
906 return false;
907 }
908
909 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
910 RxSBG.Input = N.getOperand(0);
911 return true;
912 }
913
914 case ISD::SRL:
915 case ISD::SRA: {
916 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
917 if (!CountNode)
918 return false;
919
920 uint64_t Count = CountNode->getZExtValue();
921 unsigned BitSize = N.getValueSizeInBits();
922 if (Count < 1 || Count >= BitSize)
923 return false;
924
925 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
926 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
927 // count bits from RxSBG.Input are ignored.
928 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
929 return false;
930 } else {
931 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
932 // which is similar to SLL above.
933 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
934 return false;
935 }
936
937 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
938 RxSBG.Input = N.getOperand(0);
939 return true;
940 }
941 default:
942 return false;
943 }
944}
945
946SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
947 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
948 return SDValue(N, 0);
949}
950
951SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
952 SDValue N) const {
953 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
954 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
955 DL, VT, getUNDEF(DL, MVT::i64), N);
956 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
957 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
958 assert(N.getValueType() == VT && "Unexpected value types");
959 return N;
960}
961
962bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
963 SDLoc DL(N);
964 EVT VT = N->getValueType(0);
965 if (!VT.isInteger() || VT.getSizeInBits() > 64)
966 return false;
967 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
968 unsigned Count = 0;
969 while (expandRxSBG(RISBG))
970 // The widening or narrowing is expected to be free.
971 // Counting widening or narrowing as a saved operation will result in
972 // preferring an R*SBG over a simple shift/logical instruction.
973 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
974 RISBG.Input.getOpcode() != ISD::TRUNCATE)
975 Count += 1;
976 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
977 return false;
978
979 // Prefer to use normal shift instructions over RISBG, since they can handle
980 // all cases and are sometimes shorter.
981 if (Count == 1 && N->getOpcode() != ISD::AND)
982 return false;
983
984 // Prefer register extensions like LLC over RISBG. Also prefer to start
985 // out with normal ANDs if one instruction would be enough. We can convert
986 // these ANDs into an RISBG later if a three-address instruction is useful.
987 if (RISBG.Rotate == 0) {
988 bool PreferAnd = false;
989 // Prefer AND for any 32-bit and-immediate operation.
990 if (VT == MVT::i32)
991 PreferAnd = true;
992 // As well as for any 64-bit operation that can be implemented via LLC(R),
993 // LLH(R), LLGT(R), or one of the and-immediate instructions.
994 else if (RISBG.Mask == 0xff ||
995 RISBG.Mask == 0xffff ||
996 RISBG.Mask == 0x7fffffff ||
997 SystemZ::isImmLF(~RISBG.Mask) ||
998 SystemZ::isImmHF(~RISBG.Mask))
999 PreferAnd = true;
1000 // And likewise for the LLZRGF instruction, which doesn't have a register
1001 // to register version.
1002 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1003 if (Load->getMemoryVT() == MVT::i32 &&
1004 (Load->getExtensionType() == ISD::EXTLOAD ||
1005 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1006 RISBG.Mask == 0xffffff00 &&
1007 Subtarget->hasLoadAndZeroRightmostByte())
1008 PreferAnd = true;
1009 }
1010 if (PreferAnd) {
1011 // Replace the current node with an AND. Note that the current node
1012 // might already be that same AND, in which case it is already CSE'd
1013 // with it, and we must not call ReplaceNode.
1014 SDValue In = convertTo(DL, VT, RISBG.Input);
1015 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1016 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1017 if (N != New.getNode()) {
1018 insertDAGNode(CurDAG, N, Mask);
1019 insertDAGNode(CurDAG, N, New);
1020 ReplaceNode(N, New.getNode());
1021 N = New.getNode();
1022 }
1023 // Now, select the machine opcode to implement this operation.
1024 if (!N->isMachineOpcode())
1025 SelectCode(N);
1026 return true;
1027 }
1028 }
1029
1030 unsigned Opcode = SystemZ::RISBG;
1031 // Prefer RISBGN if available, since it does not clobber CC.
1032 if (Subtarget->hasMiscellaneousExtensions())
1033 Opcode = SystemZ::RISBGN;
1034 EVT OpcodeVT = MVT::i64;
1035 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1036 // We can only use the 32-bit instructions if all source bits are
1037 // in the low 32 bits without wrapping, both after rotation (because
1038 // of the smaller range for Start and End) and before rotation
1039 // (because the input value is truncated).
1040 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1041 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1042 ((RISBG.End + RISBG.Rotate) & 63) >=
1043 ((RISBG.Start + RISBG.Rotate) & 63)) {
1044 Opcode = SystemZ::RISBMux;
1045 OpcodeVT = MVT::i32;
1046 RISBG.Start &= 31;
1047 RISBG.End &= 31;
1048 }
1049 SDValue Ops[5] = {
1050 getUNDEF(DL, OpcodeVT),
1051 convertTo(DL, OpcodeVT, RISBG.Input),
1052 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1053 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1054 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1055 };
1056 SDValue New = convertTo(
1057 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1058 ReplaceNode(N, New.getNode());
1059 return true;
1060}
1061
1062bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1063 SDLoc DL(N);
1064 EVT VT = N->getValueType(0);
1065 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1066 return false;
1067 // Try treating each operand of N as the second operand of the RxSBG
1068 // and see which goes deepest.
1069 RxSBGOperands RxSBG[] = {
1070 RxSBGOperands(Opcode, N->getOperand(0)),
1071 RxSBGOperands(Opcode, N->getOperand(1))
1072 };
1073 unsigned Count[] = { 0, 0 };
1074 for (unsigned I = 0; I < 2; ++I)
1075 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I]))
1076 // In cases of multiple users it seems better to keep the simple
1077 // instruction as they are one cycle faster, and it also helps in cases
1078 // where both inputs share a common node.
1079 // The widening or narrowing is expected to be free. Counting widening
1080 // or narrowing as a saved operation will result in preferring an R*SBG
1081 // over a simple shift/logical instruction.
1082 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1083 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1084 Count[I] += 1;
1085
1086 // Do nothing if neither operand is suitable.
1087 if (Count[0] == 0 && Count[1] == 0)
1088 return false;
1089
1090 // Pick the deepest second operand.
1091 unsigned I = Count[0] > Count[1] ? 0 : 1;
1092 SDValue Op0 = N->getOperand(I ^ 1);
1093
1094 // Prefer IC for character insertions from memory.
1095 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1096 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1097 if (Load->getMemoryVT() == MVT::i8)
1098 return false;
1099
1100 // See whether we can avoid an AND in the first operand by converting
1101 // ROSBG to RISBG.
1102 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1103 Opcode = SystemZ::RISBG;
1104 // Prefer RISBGN if available, since it does not clobber CC.
1105 if (Subtarget->hasMiscellaneousExtensions())
1106 Opcode = SystemZ::RISBGN;
1107 }
1108
1109 SDValue Ops[5] = {
1110 convertTo(DL, MVT::i64, Op0),
1111 convertTo(DL, MVT::i64, RxSBG[I].Input),
1112 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1113 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1114 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1115 };
1116 SDValue New = convertTo(
1117 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1118 ReplaceNode(N, New.getNode());
1119 return true;
1120}
1121
1122void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1123 SDValue Op0, uint64_t UpperVal,
1124 uint64_t LowerVal) {
1125 EVT VT = Node->getValueType(0);
1126 SDLoc DL(Node);
1127 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1128 if (Op0.getNode())
1129 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1130
1131 {
1132 // When we haven't passed in Op0, Upper will be a constant. In order to
1133 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1134 // SelectCode first and end up with an opaque machine node. This means that
1135 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1136 // SelectCode.
1137 //
1138 // Note that in the case where Op0 is passed in we could just call
1139 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1140 // the handle at all, but it's fine to do it here.
1141 //
1142 // TODO: This is a pretty hacky way to do this. Can we do something that
1143 // doesn't require a two paragraph explanation?
1144 HandleSDNode Handle(Upper);
1145 SelectCode(Upper.getNode());
1146 Upper = Handle.getValue();
1147 }
1148
1149 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1150 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1151
1152 ReplaceNode(Node, Or.getNode());
1153
1154 SelectCode(Or.getNode());
1155}
1156
1157void SystemZDAGToDAGISel::loadVectorConstant(
1158 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1162 "Bad opcode!");
1163 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1164 EVT VT = Node->getValueType(0);
1165 SDLoc DL(Node);
1167 for (unsigned OpVal : VCI.OpVals)
1168 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1169 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1170
1171 if (VCI.VecVT == VT.getSimpleVT())
1172 ReplaceNode(Node, Op.getNode());
1173 else if (VT.getSizeInBits() == 128) {
1174 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1175 ReplaceNode(Node, BitCast.getNode());
1176 SelectCode(BitCast.getNode());
1177 } else { // float or double
1178 unsigned SubRegIdx =
1179 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1180 ReplaceNode(
1181 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1182 }
1183 SelectCode(Op.getNode());
1184}
1185
1186bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1187 SDValue ElemV = N->getOperand(2);
1188 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1189 if (!ElemN)
1190 return false;
1191
1192 unsigned Elem = ElemN->getZExtValue();
1193 EVT VT = N->getValueType(0);
1194 if (Elem >= VT.getVectorNumElements())
1195 return false;
1196
1197 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1198 if (!Load || !Load->hasNUsesOfValue(1, 0))
1199 return false;
1200 if (Load->getMemoryVT().getSizeInBits() !=
1201 Load->getValueType(0).getSizeInBits())
1202 return false;
1203
1204 SDValue Base, Disp, Index;
1205 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1206 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1207 return false;
1208
1209 SDLoc DL(Load);
1210 SDValue Ops[] = {
1211 N->getOperand(0), Base, Disp, Index,
1212 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1213 };
1214 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1215 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1216 ReplaceNode(N, Res);
1217 return true;
1218}
1219
1220bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1221 SDValue Value = Store->getValue();
1222 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1223 return false;
1224 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1225 return false;
1226
1227 SDValue ElemV = Value.getOperand(1);
1228 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1229 if (!ElemN)
1230 return false;
1231
1232 SDValue Vec = Value.getOperand(0);
1233 EVT VT = Vec.getValueType();
1234 unsigned Elem = ElemN->getZExtValue();
1235 if (Elem >= VT.getVectorNumElements())
1236 return false;
1237
1238 SDValue Base, Disp, Index;
1239 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1240 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1241 return false;
1242
1243 SDLoc DL(Store);
1244 SDValue Ops[] = {
1245 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1246 Store->getChain()
1247 };
1248 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1249 return true;
1250}
1251
1252// Check whether or not the chain ending in StoreNode is suitable for doing
1253// the {load; op; store} to modify transformation.
1255 SDValue StoredVal, SelectionDAG *CurDAG,
1256 LoadSDNode *&LoadNode,
1257 SDValue &InputChain) {
1258 // Is the stored value result 0 of the operation?
1259 if (StoredVal.getResNo() != 0)
1260 return false;
1261
1262 // Are there other uses of the loaded value than the operation?
1263 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1264 return false;
1265
1266 // Is the store non-extending and non-indexed?
1267 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1268 return false;
1269
1270 SDValue Load = StoredVal->getOperand(0);
1271 // Is the stored value a non-extending and non-indexed load?
1272 if (!ISD::isNormalLoad(Load.getNode()))
1273 return false;
1274
1275 // Return LoadNode by reference.
1276 LoadNode = cast<LoadSDNode>(Load);
1277
1278 // Is store the only read of the loaded value?
1279 if (!Load.hasOneUse())
1280 return false;
1281
1282 // Is the address of the store the same as the load?
1283 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1284 LoadNode->getOffset() != StoreNode->getOffset())
1285 return false;
1286
1287 // Check if the chain is produced by the load or is a TokenFactor with
1288 // the load output chain as an operand. Return InputChain by reference.
1289 SDValue Chain = StoreNode->getChain();
1290
1291 bool ChainCheck = false;
1292 if (Chain == Load.getValue(1)) {
1293 ChainCheck = true;
1294 InputChain = LoadNode->getChain();
1295 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1296 SmallVector<SDValue, 4> ChainOps;
1297 SmallVector<const SDNode *, 4> LoopWorklist;
1299 const unsigned int Max = 1024;
1300 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1301 SDValue Op = Chain.getOperand(i);
1302 if (Op == Load.getValue(1)) {
1303 ChainCheck = true;
1304 // Drop Load, but keep its chain. No cycle check necessary.
1305 ChainOps.push_back(Load.getOperand(0));
1306 continue;
1307 }
1308 LoopWorklist.push_back(Op.getNode());
1309 ChainOps.push_back(Op);
1310 }
1311
1312 if (ChainCheck) {
1313 // Add the other operand of StoredVal to worklist.
1314 for (SDValue Op : StoredVal->ops())
1315 if (Op.getNode() != LoadNode)
1316 LoopWorklist.push_back(Op.getNode());
1317
1318 // Check if Load is reachable from any of the nodes in the worklist.
1319 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1320 true))
1321 return false;
1322
1323 // Make a new TokenFactor with all the other input chains except
1324 // for the load.
1325 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1326 MVT::Other, ChainOps);
1327 }
1328 }
1329 if (!ChainCheck)
1330 return false;
1331
1332 return true;
1333}
1334
1335// Change a chain of {load; op; store} of the same value into a simple op
1336// through memory of that value, if the uses of the modified value and its
1337// address are suitable.
1338//
1339// The tablegen pattern memory operand pattern is currently not able to match
1340// the case where the CC on the original operation are used.
1341//
1342// See the equivalent routine in X86ISelDAGToDAG for further comments.
1343bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1344 StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1345 SDValue StoredVal = StoreNode->getOperand(1);
1346 unsigned Opc = StoredVal->getOpcode();
1347 SDLoc DL(StoreNode);
1348
1349 // Before we try to select anything, make sure this is memory operand size
1350 // and opcode we can handle. Note that this must match the code below that
1351 // actually lowers the opcodes.
1352 EVT MemVT = StoreNode->getMemoryVT();
1353 unsigned NewOpc = 0;
1354 bool NegateOperand = false;
1355 switch (Opc) {
1356 default:
1357 return false;
1358 case SystemZISD::SSUBO:
1359 NegateOperand = true;
1360 [[fallthrough]];
1361 case SystemZISD::SADDO:
1362 if (MemVT == MVT::i32)
1363 NewOpc = SystemZ::ASI;
1364 else if (MemVT == MVT::i64)
1365 NewOpc = SystemZ::AGSI;
1366 else
1367 return false;
1368 break;
1369 case SystemZISD::USUBO:
1370 NegateOperand = true;
1371 [[fallthrough]];
1372 case SystemZISD::UADDO:
1373 if (MemVT == MVT::i32)
1374 NewOpc = SystemZ::ALSI;
1375 else if (MemVT == MVT::i64)
1376 NewOpc = SystemZ::ALGSI;
1377 else
1378 return false;
1379 break;
1380 }
1381
1382 LoadSDNode *LoadNode = nullptr;
1383 SDValue InputChain;
1384 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1385 InputChain))
1386 return false;
1387
1388 SDValue Operand = StoredVal.getOperand(1);
1389 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1390 if (!OperandC)
1391 return false;
1392 auto OperandV = OperandC->getAPIntValue();
1393 if (NegateOperand)
1394 OperandV = -OperandV;
1395 if (OperandV.getSignificantBits() > 8)
1396 return false;
1397 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1398
1399 SDValue Base, Disp;
1400 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1401 return false;
1402
1403 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1405 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1406 CurDAG->setNodeMemRefs(
1407 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1408
1409 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1410 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1411 CurDAG->RemoveDeadNode(Node);
1412 return true;
1413}
1414
1415bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1416 LoadSDNode *Load) const {
1417 // Check that the two memory operands have the same size.
1418 if (Load->getMemoryVT() != Store->getMemoryVT())
1419 return false;
1420
1421 // Volatility stops an access from being decomposed.
1422 if (Load->isVolatile() || Store->isVolatile())
1423 return false;
1424
1425 // There's no chance of overlap if the load is invariant.
1426 if (Load->isInvariant() && Load->isDereferenceable())
1427 return true;
1428
1429 // Otherwise we need to check whether there's an alias.
1430 const Value *V1 = Load->getMemOperand()->getValue();
1431 const Value *V2 = Store->getMemOperand()->getValue();
1432 if (!V1 || !V2)
1433 return false;
1434
1435 // Reject equality.
1436 uint64_t Size = Load->getMemoryVT().getStoreSize();
1437 int64_t End1 = Load->getSrcValueOffset() + Size;
1438 int64_t End2 = Store->getSrcValueOffset() + Size;
1439 if (V1 == V2 && End1 == End2)
1440 return false;
1441
1442 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
1443 MemoryLocation(V2, End2, Store->getAAInfo()));
1444}
1445
1446bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1447 auto *Store = cast<StoreSDNode>(N);
1448 auto *Load = cast<LoadSDNode>(Store->getValue());
1449
1450 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1451 // instructions.
1452 uint64_t Size = Load->getMemoryVT().getStoreSize();
1453 if (Size > 1 && Size <= 8) {
1454 // Prefer LHRL, LRL and LGRL.
1455 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1456 return false;
1457 // Prefer STHRL, STRL and STGRL.
1458 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1459 return false;
1460 }
1461
1462 return canUseBlockOperation(Store, Load);
1463}
1464
1465bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1466 unsigned I) const {
1467 auto *StoreA = cast<StoreSDNode>(N);
1468 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1469 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1470 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1471 canUseBlockOperation(StoreA, LoadB);
1472}
1473
1474bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1475
1476 auto *MemAccess = cast<LSBaseSDNode>(N);
1477 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1478 SDValue BasePtr = MemAccess->getBasePtr();
1479 MachineMemOperand *MMO = MemAccess->getMemOperand();
1480 assert(MMO && "Expected a memory operand.");
1481
1482 // The memory access must have a proper alignment and no index register.
1483 if (MemAccess->getAlign().value() < StoreSize ||
1484 !MemAccess->getOffset().isUndef())
1485 return false;
1486
1487 // The MMO must not have an unaligned offset.
1488 if (MMO->getOffset() % StoreSize != 0)
1489 return false;
1490
1491 // An access to GOT or the Constant Pool is aligned.
1492 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1493 if ((PSV->isGOT() || PSV->isConstantPool()))
1494 return true;
1495
1496 // Check the alignment of a Global Address.
1497 if (BasePtr.getNumOperands())
1498 if (GlobalAddressSDNode *GA =
1499 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) {
1500 // The immediate offset must be aligned.
1501 if (GA->getOffset() % StoreSize != 0)
1502 return false;
1503
1504 // The alignment of the symbol itself must be at least the store size.
1505 const GlobalValue *GV = GA->getGlobal();
1506 const DataLayout &DL = GV->getParent()->getDataLayout();
1507 if (GV->getPointerAlignment(DL).value() < StoreSize)
1508 return false;
1509 }
1510
1511 return true;
1512}
1513
1514void SystemZDAGToDAGISel::Select(SDNode *Node) {
1515 // If we have a custom node, we already have selected!
1516 if (Node->isMachineOpcode()) {
1517 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1518 Node->setNodeId(-1);
1519 return;
1520 }
1521
1522 unsigned Opcode = Node->getOpcode();
1523 switch (Opcode) {
1524 case ISD::OR:
1525 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1526 if (tryRxSBG(Node, SystemZ::ROSBG))
1527 return;
1528 goto or_xor;
1529
1530 case ISD::XOR:
1531 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1532 if (tryRxSBG(Node, SystemZ::RXSBG))
1533 return;
1534 // Fall through.
1535 or_xor:
1536 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1537 // split the operation into two. If both operands here happen to be
1538 // constant, leave this to common code to optimize.
1539 if (Node->getValueType(0) == MVT::i64 &&
1540 Node->getOperand(0).getOpcode() != ISD::Constant)
1541 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1542 uint64_t Val = Op1->getZExtValue();
1543 // Don't split the operation if we can match one of the combined
1544 // logical operations provided by miscellaneous-extensions-3.
1545 if (Subtarget->hasMiscellaneousExtensions3()) {
1546 unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1547 // Check whether this expression matches NAND/NOR/NXOR.
1548 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1549 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1550 ChildOpcode == ISD::XOR)
1551 break;
1552 // Check whether this expression matches OR-with-complement
1553 // (or matches an alternate pattern for NXOR).
1554 if (ChildOpcode == ISD::XOR) {
1555 auto Op0 = Node->getOperand(0);
1556 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1557 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1558 break;
1559 }
1560 }
1561 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1562 splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1563 Val - uint32_t(Val), uint32_t(Val));
1564 return;
1565 }
1566 }
1567 break;
1568
1569 case ISD::AND:
1570 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1571 if (tryRxSBG(Node, SystemZ::RNSBG))
1572 return;
1573 [[fallthrough]];
1574 case ISD::ROTL:
1575 case ISD::SHL:
1576 case ISD::SRL:
1577 case ISD::ZERO_EXTEND:
1578 if (tryRISBGZero(Node))
1579 return;
1580 break;
1581
1582 case ISD::Constant:
1583 // If this is a 64-bit constant that is out of the range of LLILF,
1584 // LLIHF and LGFI, split it into two 32-bit pieces.
1585 if (Node->getValueType(0) == MVT::i64) {
1586 uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1587 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1588 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1589 uint32_t(Val));
1590 return;
1591 }
1592 }
1593 break;
1594
1596 SDValue Op0 = Node->getOperand(0);
1597 SDValue Op1 = Node->getOperand(1);
1598 // Prefer to put any load first, so that it can be matched as a
1599 // conditional load. Likewise for constants in range for LOCHI.
1600 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1601 (Subtarget->hasLoadStoreOnCond2() &&
1602 Node->getValueType(0).isInteger() &&
1603 Op1.getOpcode() == ISD::Constant &&
1604 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1605 !(Op0.getOpcode() == ISD::Constant &&
1606 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1607 SDValue CCValid = Node->getOperand(2);
1608 SDValue CCMask = Node->getOperand(3);
1609 uint64_t ConstCCValid =
1610 cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1611 uint64_t ConstCCMask =
1612 cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1613 // Invert the condition.
1614 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1615 SDLoc(Node), CCMask.getValueType());
1616 SDValue Op4 = Node->getOperand(4);
1617 SDNode *UpdatedNode =
1618 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1619 if (UpdatedNode != Node) {
1620 // In case this node already exists then replace Node with it.
1621 ReplaceNode(Node, UpdatedNode);
1622 Node = UpdatedNode;
1623 }
1624 }
1625 break;
1626 }
1627
1629 EVT VT = Node->getValueType(0);
1630 unsigned ElemBitSize = VT.getScalarSizeInBits();
1631 if (ElemBitSize == 32) {
1632 if (tryGather(Node, SystemZ::VGEF))
1633 return;
1634 } else if (ElemBitSize == 64) {
1635 if (tryGather(Node, SystemZ::VGEG))
1636 return;
1637 }
1638 break;
1639 }
1640
1641 case ISD::BUILD_VECTOR: {
1642 auto *BVN = cast<BuildVectorSDNode>(Node);
1644 if (VCI.isVectorConstantLegal(*Subtarget)) {
1645 loadVectorConstant(VCI, Node);
1646 return;
1647 }
1648 break;
1649 }
1650
1651 case ISD::ConstantFP: {
1652 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1653 if (Imm.isZero() || Imm.isNegZero())
1654 break;
1656 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1657 assert(Success && "Expected legal FP immediate");
1658 loadVectorConstant(VCI, Node);
1659 return;
1660 }
1661
1662 case ISD::STORE: {
1663 if (tryFoldLoadStoreIntoMemOperand(Node))
1664 return;
1665 auto *Store = cast<StoreSDNode>(Node);
1666 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1667 if (ElemBitSize == 32) {
1668 if (tryScatter(Store, SystemZ::VSCEF))
1669 return;
1670 } else if (ElemBitSize == 64) {
1671 if (tryScatter(Store, SystemZ::VSCEG))
1672 return;
1673 }
1674 break;
1675 }
1676 }
1677
1678 SelectCode(Node);
1679}
1680
1681bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1682 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
1683 std::vector<SDValue> &OutOps) {
1684 SystemZAddressingMode::AddrForm Form;
1685 SystemZAddressingMode::DispRange DispRange;
1686 SDValue Base, Disp, Index;
1687
1688 switch(ConstraintID) {
1689 default:
1690 llvm_unreachable("Unexpected asm memory constraint");
1691 case InlineAsm::ConstraintCode::i:
1692 case InlineAsm::ConstraintCode::Q:
1693 case InlineAsm::ConstraintCode::ZQ:
1694 // Accept an address with a short displacement, but no index.
1695 Form = SystemZAddressingMode::FormBD;
1696 DispRange = SystemZAddressingMode::Disp12Only;
1697 break;
1698 case InlineAsm::ConstraintCode::R:
1699 case InlineAsm::ConstraintCode::ZR:
1700 // Accept an address with a short displacement and an index.
1701 Form = SystemZAddressingMode::FormBDXNormal;
1702 DispRange = SystemZAddressingMode::Disp12Only;
1703 break;
1704 case InlineAsm::ConstraintCode::S:
1705 case InlineAsm::ConstraintCode::ZS:
1706 // Accept an address with a long displacement, but no index.
1707 Form = SystemZAddressingMode::FormBD;
1708 DispRange = SystemZAddressingMode::Disp20Only;
1709 break;
1710 case InlineAsm::ConstraintCode::T:
1711 case InlineAsm::ConstraintCode::m:
1712 case InlineAsm::ConstraintCode::o:
1713 case InlineAsm::ConstraintCode::p:
1714 case InlineAsm::ConstraintCode::ZT:
1715 // Accept an address with a long displacement and an index.
1716 // m works the same as T, as this is the most general case.
1717 // We don't really have any special handling of "offsettable"
1718 // memory addresses, so just treat o the same as m.
1719 Form = SystemZAddressingMode::FormBDXNormal;
1720 DispRange = SystemZAddressingMode::Disp20Only;
1721 break;
1722 }
1723
1724 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1725 const TargetRegisterClass *TRC =
1726 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1727 SDLoc DL(Base);
1728 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1729
1730 // Make sure that the base address doesn't go into %r0.
1731 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1732 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1733 Base.getOpcode() != ISD::Register) {
1734 Base =
1735 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1736 DL, Base.getValueType(),
1737 Base, RC), 0);
1738 }
1739
1740 // Make sure that the index register isn't assigned to %r0 either.
1741 if (Index.getOpcode() != ISD::Register) {
1742 Index =
1743 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1744 DL, Index.getValueType(),
1745 Index, RC), 0);
1746 }
1747
1748 OutOps.push_back(Base);
1749 OutOps.push_back(Disp);
1750 OutOps.push_back(Index);
1751 return false;
1752 }
1753
1754 return true;
1755}
1756
1757// IsProfitableToFold - Returns true if is profitable to fold the specific
1758// operand node N of U during instruction selection that starts at Root.
1759bool
1760SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1761 SDNode *Root) const {
1762 // We want to avoid folding a LOAD into an ICMP node if as a result
1763 // we would be forced to spill the condition code into a GPR.
1764 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1765 if (!N.hasOneUse() || !U->hasOneUse())
1766 return false;
1767
1768 // The user of the CC value will usually be a CopyToReg into the
1769 // physical CC register, which in turn is glued and chained to the
1770 // actual instruction that uses the CC value. Bail out if we have
1771 // anything else than that.
1772 SDNode *CCUser = *U->use_begin();
1773 SDNode *CCRegUser = nullptr;
1774 if (CCUser->getOpcode() == ISD::CopyToReg ||
1775 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1776 for (auto *U : CCUser->uses()) {
1777 if (CCRegUser == nullptr)
1778 CCRegUser = U;
1779 else if (CCRegUser != U)
1780 return false;
1781 }
1782 }
1783 if (CCRegUser == nullptr)
1784 return false;
1785
1786 // If the actual instruction is a branch, the only thing that remains to be
1787 // checked is whether the CCUser chain is a predecessor of the load.
1788 if (CCRegUser->isMachineOpcode() &&
1789 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1790 return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1791
1792 // Otherwise, the instruction may have multiple operands, and we need to
1793 // verify that none of them are a predecessor of the load. This is exactly
1794 // the same check that would be done by common code if the CC setter were
1795 // glued to the CC user, so simply invoke that check here.
1796 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1797 return false;
1798 }
1799
1800 return true;
1801}
1802
1803namespace {
1804// Represents a sequence for extracting a 0/1 value from an IPM result:
1805// (((X ^ XORValue) + AddValue) >> Bit)
1806struct IPMConversion {
1807 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1808 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1809
1810 int64_t XORValue;
1811 int64_t AddValue;
1812 unsigned Bit;
1813};
1814} // end anonymous namespace
1815
1816// Return a sequence for getting a 1 from an IPM result when CC has a
1817// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1818// The handling of CC values outside CCValid doesn't matter.
1819static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1820 // Deal with cases where the result can be taken directly from a bit
1821 // of the IPM result.
1822 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1823 return IPMConversion(0, 0, SystemZ::IPM_CC);
1824 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1825 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1826
1827 // Deal with cases where we can add a value to force the sign bit
1828 // to contain the right value. Putting the bit in 31 means we can
1829 // use SRL rather than RISBG(L), and also makes it easier to get a
1830 // 0/-1 value, so it has priority over the other tests below.
1831 //
1832 // These sequences rely on the fact that the upper two bits of the
1833 // IPM result are zero.
1834 uint64_t TopBit = uint64_t(1) << 31;
1835 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1836 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1837 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1838 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1839 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1842 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1843 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1844 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1845 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1848 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1849
1850 // Next try inverting the value and testing a bit. 0/1 could be
1851 // handled this way too, but we dealt with that case above.
1852 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1853 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1854
1855 // Handle cases where adding a value forces a non-sign bit to contain
1856 // the right value.
1857 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1858 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1859 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1860 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1861
1862 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1863 // can be done by inverting the low CC bit and applying one of the
1864 // sign-based extractions above.
1865 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1866 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1867 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1868 return IPMConversion(1 << SystemZ::IPM_CC,
1869 TopBit - (3 << SystemZ::IPM_CC), 31);
1870 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1873 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1874 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1877 return IPMConversion(1 << SystemZ::IPM_CC,
1878 TopBit - (1 << SystemZ::IPM_CC), 31);
1879
1880 llvm_unreachable("Unexpected CC combination");
1881}
1882
1883SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1884 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1885 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1886 if (!TrueOp || !FalseOp)
1887 return SDValue();
1888 if (FalseOp->getZExtValue() != 0)
1889 return SDValue();
1890 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1891 return SDValue();
1892
1893 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1894 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1895 if (!CCValidOp || !CCMaskOp)
1896 return SDValue();
1897 int CCValid = CCValidOp->getZExtValue();
1898 int CCMask = CCMaskOp->getZExtValue();
1899
1900 SDLoc DL(Node);
1901 SDValue CCReg = Node->getOperand(4);
1902 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1903 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1904
1905 if (IPM.XORValue)
1906 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1907 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1908
1909 if (IPM.AddValue)
1910 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1911 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1912
1913 EVT VT = Node->getValueType(0);
1914 if (VT == MVT::i32 && IPM.Bit == 31) {
1915 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
1916 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1917 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1918 } else {
1919 if (VT != MVT::i32)
1920 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1921
1922 if (TrueOp->getSExtValue() == 1) {
1923 // The SHR/AND sequence should get optimized to an RISBG.
1924 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1925 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1926 Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1927 CurDAG->getConstant(1, DL, VT));
1928 } else {
1929 // Sign-extend from IPM.Bit using a pair of shifts.
1930 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
1931 int SraAmt = VT.getSizeInBits() - 1;
1932 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
1933 CurDAG->getConstant(ShlAmt, DL, MVT::i32));
1934 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
1935 CurDAG->getConstant(SraAmt, DL, MVT::i32));
1936 }
1937 }
1938
1939 return Result;
1940}
1941
1942void SystemZDAGToDAGISel::PreprocessISelDAG() {
1943 // If we have conditional immediate loads, we always prefer
1944 // using those over an IPM sequence.
1945 if (Subtarget->hasLoadStoreOnCond2())
1946 return;
1947
1948 bool MadeChange = false;
1949
1950 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
1951 E = CurDAG->allnodes_end();
1952 I != E;) {
1953 SDNode *N = &*I++;
1954 if (N->use_empty())
1955 continue;
1956
1957 SDValue Res;
1958 switch (N->getOpcode()) {
1959 default: break;
1961 Res = expandSelectBoolean(N);
1962 break;
1963 }
1964
1965 if (Res) {
1966 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
1967 LLVM_DEBUG(N->dump(CurDAG));
1968 LLVM_DEBUG(dbgs() << "\nNew: ");
1969 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
1970 LLVM_DEBUG(dbgs() << "\n");
1971
1972 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1973 MadeChange = true;
1974 }
1975 }
1976
1977 if (MadeChange)
1978 CurDAG->RemoveDeadNodes();
1979}
#define Success
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
bool End
Definition: ELF_riscv.cpp:478
const HexagonInstrInfo * TII
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)
#define PASS_NAME
#define DEBUG_TYPE
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static uint64_t allOnes(unsigned int Count)
static constexpr uint32_t Opcode
Definition: aarch32.h:200
DEMANGLE_DUMP_METHOD void dump() const
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1485
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
An SDNode that represents everything that will be needed to construct a MachineInstr.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isNonTemporal() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:275
Special value supplied for machine level alias analysis.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
int getNodeId() const
Return the unique node id.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:534
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:451
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SystemZInstrInfo * getInstrInfo() const override
unsigned getID() const
Return the register class ID number.
Target - Wrapper for Target specific information.
LLVM Value Representation.
Definition: Value.h:74
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition: Value.cpp:926
self_iterator getIterator()
Definition: ilist_node.h:109
#define UINT64_MAX
Definition: DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1029
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ FrameIndex
Definition: ISDOpcodes.h:80
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ TargetFrameIndex
Definition: ISDOpcodes.h:166
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isPCREL(unsigned Opcode)
const unsigned CCMASK_0
Definition: SystemZ.h:27
const unsigned CCMASK_1
Definition: SystemZ.h:28
const unsigned IPM_CC
Definition: SystemZ.h:112
static bool isImmHF(uint64_t Val)
Definition: SystemZ.h:186
const unsigned CCMASK_3
Definition: SystemZ.h:30
static bool isImmLF(uint64_t Val)
Definition: SystemZ.h:181
const unsigned CCMASK_2
Definition: SystemZ.h:29
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition: DWP.cpp:456
FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Or
Bitwise or logical OR of integers.
#define N
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:93
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:363
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
SmallVector< unsigned, 2 > OpVals
bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)