LLVM  16.0.0git
PPCISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines a pattern matching instruction selector for PowerPC,
10 // converting from a legalized dag to a PPC dag.
11 //
12 //===----------------------------------------------------------------------===//
13 
16 #include "PPC.h"
17 #include "PPCISelLowering.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCSubtarget.h"
20 #include "PPCTargetMachine.h"
21 #include "llvm/ADT/APInt.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/Statistic.h"
41 #include "llvm/IR/BasicBlock.h"
42 #include "llvm/IR/DebugLoc.h"
43 #include "llvm/IR/Function.h"
44 #include "llvm/IR/GlobalValue.h"
45 #include "llvm/IR/InlineAsm.h"
46 #include "llvm/IR/InstrTypes.h"
47 #include "llvm/IR/IntrinsicsPowerPC.h"
48 #include "llvm/IR/Module.h"
49 #include "llvm/Support/Casting.h"
50 #include "llvm/Support/CodeGen.h"
52 #include "llvm/Support/Compiler.h"
53 #include "llvm/Support/Debug.h"
55 #include "llvm/Support/KnownBits.h"
59 #include <algorithm>
60 #include <cassert>
61 #include <cstdint>
62 #include <iterator>
63 #include <limits>
64 #include <memory>
65 #include <new>
66 #include <tuple>
67 #include <utility>
68 
69 using namespace llvm;
70 
71 #define DEBUG_TYPE "ppc-codegen"
72 
73 STATISTIC(NumSextSetcc,
74  "Number of (sext(setcc)) nodes expanded into GPR sequence.");
75 STATISTIC(NumZextSetcc,
76  "Number of (zext(setcc)) nodes expanded into GPR sequence.");
77 STATISTIC(SignExtensionsAdded,
78  "Number of sign extensions for compare inputs added.");
79 STATISTIC(ZeroExtensionsAdded,
80  "Number of zero extensions for compare inputs added.");
81 STATISTIC(NumLogicOpsOnComparison,
82  "Number of logical ops on i1 values calculated in GPR.");
83 STATISTIC(OmittedForNonExtendUses,
84  "Number of compares not eliminated as they have non-extending uses.");
85 STATISTIC(NumP9Setb,
86  "Number of compares lowered to setb.");
87 
88 // FIXME: Remove this once the bug has been fixed!
89 cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug",
90 cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden);
91 
92 static cl::opt<bool>
93  UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true),
94  cl::desc("use aggressive ppc isel for bit permutations"),
95  cl::Hidden);
97  "ppc-bit-perm-rewriter-stress-rotates",
98  cl::desc("stress rotate selection in aggressive ppc isel for "
99  "bit permutations"),
100  cl::Hidden);
101 
103  "ppc-use-branch-hint", cl::init(true),
104  cl::desc("Enable static hinting of branches on ppc"),
105  cl::Hidden);
106 
108  "ppc-tls-opt", cl::init(true),
109  cl::desc("Enable tls optimization peephole"),
110  cl::Hidden);
111 
115 
117  "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All),
118  cl::desc("Specify the types of comparisons to emit GPR-only code for."),
119  cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."),
120  clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."),
121  clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."),
122  clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."),
123  clEnumValN(ICGPR_NonExtIn, "nonextin",
124  "Only comparisons where inputs don't need [sz]ext."),
125  clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."),
126  clEnumValN(ICGPR_ZextI32, "zexti32",
127  "Only i32 comparisons with zext result."),
128  clEnumValN(ICGPR_ZextI64, "zexti64",
129  "Only i64 comparisons with zext result."),
130  clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."),
131  clEnumValN(ICGPR_SextI32, "sexti32",
132  "Only i32 comparisons with sext result."),
133  clEnumValN(ICGPR_SextI64, "sexti64",
134  "Only i64 comparisons with sext result.")));
135 namespace {
136 
137  //===--------------------------------------------------------------------===//
138  /// PPCDAGToDAGISel - PPC specific code to select PPC machine
139  /// instructions for SelectionDAG operations.
140  ///
141  class PPCDAGToDAGISel : public SelectionDAGISel {
142  const PPCTargetMachine &TM;
143  const PPCSubtarget *Subtarget = nullptr;
144  const PPCTargetLowering *PPCLowering = nullptr;
145  unsigned GlobalBaseReg = 0;
146 
147  public:
148  explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel)
149  : SelectionDAGISel(tm, OptLevel), TM(tm) {}
150 
151  bool runOnMachineFunction(MachineFunction &MF) override {
152  // Make sure we re-emit a set of the global base reg if necessary
153  GlobalBaseReg = 0;
154  Subtarget = &MF.getSubtarget<PPCSubtarget>();
155  PPCLowering = Subtarget->getTargetLowering();
156  if (Subtarget->hasROPProtect()) {
157  // Create a place on the stack for the ROP Protection Hash.
158  // The ROP Protection Hash will always be 8 bytes and aligned to 8
159  // bytes.
160  MachineFrameInfo &MFI = MF.getFrameInfo();
162  const int Result = MFI.CreateStackObject(8, Align(8), false);
163  FI->setROPProtectionHashSaveIndex(Result);
164  }
166 
167  return true;
168  }
169 
170  void PreprocessISelDAG() override;
171  void PostprocessISelDAG() override;
172 
173  /// getI16Imm - Return a target constant with the specified value, of type
174  /// i16.
175  inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) {
176  return CurDAG->getTargetConstant(Imm, dl, MVT::i16);
177  }
178 
179  /// getI32Imm - Return a target constant with the specified value, of type
180  /// i32.
181  inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
182  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
183  }
184 
185  /// getI64Imm - Return a target constant with the specified value, of type
186  /// i64.
187  inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) {
188  return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
189  }
190 
191  /// getSmallIPtrImm - Return a target constant of pointer type.
192  inline SDValue getSmallIPtrImm(uint64_t Imm, const SDLoc &dl) {
193  return CurDAG->getTargetConstant(
194  Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
195  }
196 
197  /// isRotateAndMask - Returns true if Mask and Shift can be folded into a
198  /// rotate and mask opcode and mask operation.
199  static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask,
200  unsigned &SH, unsigned &MB, unsigned &ME);
201 
202  /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC
203  /// base register. Return the virtual register that holds this value.
204  SDNode *getGlobalBaseReg();
205 
206  void selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset = 0);
207 
208  // Select - Convert the specified operand from a target-independent to a
209  // target-specific node if it hasn't already been changed.
210  void Select(SDNode *N) override;
211 
212  bool tryBitfieldInsert(SDNode *N);
213  bool tryBitPermutation(SDNode *N);
214  bool tryIntCompareInGPR(SDNode *N);
215 
216  // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into
217  // an X-Form load instruction with the offset being a relocation coming from
218  // the PPCISD::ADD_TLS.
219  bool tryTLSXFormLoad(LoadSDNode *N);
220  // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into
221  // an X-Form store instruction with the offset being a relocation coming from
222  // the PPCISD::ADD_TLS.
223  bool tryTLSXFormStore(StoreSDNode *N);
224  /// SelectCC - Select a comparison of the specified values with the
225  /// specified condition code, returning the CR# of the expression.
227  const SDLoc &dl, SDValue Chain = SDValue());
228 
229  /// SelectAddrImmOffs - Return true if the operand is valid for a preinc
230  /// immediate field. Note that the operand at this point is already the
231  /// result of a prior SelectAddressRegImm call.
232  bool SelectAddrImmOffs(SDValue N, SDValue &Out) const {
233  if (N.getOpcode() == ISD::TargetConstant ||
234  N.getOpcode() == ISD::TargetGlobalAddress) {
235  Out = N;
236  return true;
237  }
238 
239  return false;
240  }
241 
242  /// SelectDSForm - Returns true if address N can be represented by the
243  /// addressing mode of DSForm instructions (a base register, plus a signed
244  /// 16-bit displacement that is a multiple of 4.
245  bool SelectDSForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
246  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
247  Align(4)) == PPC::AM_DSForm;
248  }
249 
250  /// SelectDQForm - Returns true if address N can be represented by the
251  /// addressing mode of DQForm instructions (a base register, plus a signed
252  /// 16-bit displacement that is a multiple of 16.
253  bool SelectDQForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
254  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
255  Align(16)) == PPC::AM_DQForm;
256  }
257 
258  /// SelectDForm - Returns true if address N can be represented by
259  /// the addressing mode of DForm instructions (a base register, plus a
260  /// signed 16-bit immediate.
261  bool SelectDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
262  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
263  None) == PPC::AM_DForm;
264  }
265 
266  /// SelectPCRelForm - Returns true if address N can be represented by
267  /// PC-Relative addressing mode.
268  bool SelectPCRelForm(SDNode *Parent, SDValue N, SDValue &Disp,
269  SDValue &Base) {
270  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
271  None) == PPC::AM_PCRel;
272  }
273 
274  /// SelectPDForm - Returns true if address N can be represented by Prefixed
275  /// DForm addressing mode (a base register, plus a signed 34-bit immediate.
276  bool SelectPDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
277  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
279  }
280 
281  /// SelectXForm - Returns true if address N can be represented by the
282  /// addressing mode of XForm instructions (an indexed [r+r] operation).
283  bool SelectXForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
284  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
285  None) == PPC::AM_XForm;
286  }
287 
288  /// SelectForceXForm - Given the specified address, force it to be
289  /// represented as an indexed [r+r] operation (an XForm instruction).
290  bool SelectForceXForm(SDNode *Parent, SDValue N, SDValue &Disp,
291  SDValue &Base) {
292  return PPCLowering->SelectForceXFormMode(N, Disp, Base, *CurDAG) ==
294  }
295 
296  /// SelectAddrIdx - Given the specified address, check to see if it can be
297  /// represented as an indexed [r+r] operation.
298  /// This is for xform instructions whose associated displacement form is D.
299  /// The last parameter \p 0 means associated D form has no requirment for 16
300  /// bit signed displacement.
301  /// Returns false if it can be represented by [r+imm], which are preferred.
302  bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
303  return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, None);
304  }
305 
306  /// SelectAddrIdx4 - Given the specified address, check to see if it can be
307  /// represented as an indexed [r+r] operation.
308  /// This is for xform instructions whose associated displacement form is DS.
309  /// The last parameter \p 4 means associated DS form 16 bit signed
310  /// displacement must be a multiple of 4.
311  /// Returns false if it can be represented by [r+imm], which are preferred.
312  bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
313  return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
314  Align(4));
315  }
316 
317  /// SelectAddrIdx16 - Given the specified address, check to see if it can be
318  /// represented as an indexed [r+r] operation.
319  /// This is for xform instructions whose associated displacement form is DQ.
320  /// The last parameter \p 16 means associated DQ form 16 bit signed
321  /// displacement must be a multiple of 16.
322  /// Returns false if it can be represented by [r+imm], which are preferred.
323  bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
324  return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
325  Align(16));
326  }
327 
328  /// SelectAddrIdxOnly - Given the specified address, force it to be
329  /// represented as an indexed [r+r] operation.
330  bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
331  return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG);
332  }
333 
334  /// SelectAddrImm - Returns true if the address N can be represented by
335  /// a base register plus a signed 16-bit displacement [r+imm].
336  /// The last parameter \p 0 means D form has no requirment for 16 bit signed
337  /// displacement.
338  bool SelectAddrImm(SDValue N, SDValue &Disp,
339  SDValue &Base) {
340  return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, None);
341  }
342 
343  /// SelectAddrImmX4 - Returns true if the address N can be represented by
344  /// a base register plus a signed 16-bit displacement that is a multiple of
345  /// 4 (last parameter). Suitable for use by STD and friends.
346  bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
347  return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4));
348  }
349 
350  /// SelectAddrImmX16 - Returns true if the address N can be represented by
351  /// a base register plus a signed 16-bit displacement that is a multiple of
352  /// 16(last parameter). Suitable for use by STXV and friends.
353  bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
354  return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG,
355  Align(16));
356  }
357 
358  /// SelectAddrImmX34 - Returns true if the address N can be represented by
359  /// a base register plus a signed 34-bit displacement. Suitable for use by
360  /// PSTXVP and friends.
361  bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) {
362  return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG);
363  }
364 
365  // Select an address into a single register.
366  bool SelectAddr(SDValue N, SDValue &Base) {
367  Base = N;
368  return true;
369  }
370 
371  bool SelectAddrPCRel(SDValue N, SDValue &Base) {
372  return PPCLowering->SelectAddressPCRel(N, Base);
373  }
374 
375  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
376  /// inline asm expressions. It is always correct to compute the value into
377  /// a register. The case of adding a (possibly relocatable) constant to a
378  /// register can be improved, but it is wrong to substitute Reg+Reg for
379  /// Reg in an asm, because the load or store opcode would have to change.
380  bool SelectInlineAsmMemoryOperand(const SDValue &Op,
381  unsigned ConstraintID,
382  std::vector<SDValue> &OutOps) override {
383  switch(ConstraintID) {
384  default:
385  errs() << "ConstraintID: " << ConstraintID << "\n";
386  llvm_unreachable("Unexpected asm memory constraint");
393  // We need to make sure that this one operand does not end up in r0
394  // (because we might end up lowering this as 0(%op)).
395  const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
396  const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1);
397  SDLoc dl(Op);
398  SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
399  SDValue NewOp =
400  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
401  dl, Op.getValueType(),
402  Op, RC), 0);
403 
404  OutOps.push_back(NewOp);
405  return false;
406  }
407  return true;
408  }
409 
410  StringRef getPassName() const override {
411  return "PowerPC DAG->DAG Pattern Instruction Selection";
412  }
413 
414 // Include the pieces autogenerated from the target description.
415 #include "PPCGenDAGISel.inc"
416 
417 private:
418  bool trySETCC(SDNode *N);
419  bool tryFoldSWTestBRCC(SDNode *N);
420  bool trySelectLoopCountIntrinsic(SDNode *N);
421  bool tryAsSingleRLDICL(SDNode *N);
422  bool tryAsSingleRLDICR(SDNode *N);
423  bool tryAsSingleRLWINM(SDNode *N);
424  bool tryAsSingleRLWINM8(SDNode *N);
425  bool tryAsSingleRLWIMI(SDNode *N);
426  bool tryAsPairOfRLDICL(SDNode *N);
427  bool tryAsSingleRLDIMI(SDNode *N);
428 
429  void PeepholePPC64();
430  void PeepholePPC64ZExt();
431  void PeepholeCROps();
432 
433  SDValue combineToCMPB(SDNode *N);
434  void foldBoolExts(SDValue &Res, SDNode *&N);
435 
436  bool AllUsersSelectZero(SDNode *N);
437  void SwapAllSelectUsers(SDNode *N);
438 
439  bool isOffsetMultipleOf(SDNode *N, unsigned Val) const;
440  void transferMemOperands(SDNode *N, SDNode *Result);
441  };
442 
443 } // end anonymous namespace
444 
445 /// getGlobalBaseReg - Output the instructions required to put the
446 /// base address to use for accessing globals into a register.
447 ///
448 SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
449  if (!GlobalBaseReg) {
450  const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
451  // Insert the set of GlobalBaseReg into the first MBB of the function
452  MachineBasicBlock &FirstMBB = MF->front();
454  const Module *M = MF->getFunction().getParent();
455  DebugLoc dl;
456 
457  if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) {
458  if (Subtarget->isTargetELF()) {
459  GlobalBaseReg = PPC::R30;
460  if (!Subtarget->isSecurePlt() &&
461  M->getPICLevel() == PICLevel::SmallPIC) {
462  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR));
463  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
464  MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
465  } else {
466  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
467  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
468  Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
469  BuildMI(FirstMBB, MBBI, dl,
470  TII.get(PPC::UpdateGBR), GlobalBaseReg)
472  MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
473  }
474  } else {
475  GlobalBaseReg =
476  RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass);
477  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
478  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
479  }
480  } else {
481  // We must ensure that this sequence is dominated by the prologue.
482  // FIXME: This is a bit of a big hammer since we don't get the benefits
483  // of shrink-wrapping whenever we emit this instruction. Considering
484  // this is used in any function where we emit a jump table, this may be
485  // a significant limitation. We should consider inserting this in the
486  // block where it is used and then commoning this sequence up if it
487  // appears in multiple places.
488  // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of
489  // MovePCtoLR8.
490  MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true);
491  GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
492  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8));
493  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg);
494  }
495  }
496  return CurDAG->getRegister(GlobalBaseReg,
497  PPCLowering->getPointerTy(CurDAG->getDataLayout()))
498  .getNode();
499 }
500 
501 // Check if a SDValue has the toc-data attribute.
502 static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) {
503  GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val);
504  if (!GA)
505  return false;
506 
507  const GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(GA->getGlobal());
508  if (!GV)
509  return false;
510 
511  if (!GV->hasAttribute("toc-data"))
512  return false;
513 
514  // TODO: These asserts should be updated as more support for the toc data
515  // transformation is added (struct support, etc.).
516 
517  assert(
518  PointerSize >= GV->getAlign().valueOrOne().value() &&
519  "GlobalVariables with an alignment requirement stricter than TOC entry "
520  "size not supported by the toc data transformation.");
521 
522  Type *GVType = GV->getValueType();
523 
524  assert(GVType->isSized() && "A GlobalVariable's size must be known to be "
525  "supported by the toc data transformation.");
526 
527  if (GVType->isVectorTy())
528  report_fatal_error("A GlobalVariable of Vector type is not currently "
529  "supported by the toc data transformation.");
530 
531  if (GVType->isArrayTy())
532  report_fatal_error("A GlobalVariable of Array type is not currently "
533  "supported by the toc data transformation.");
534 
535  if (GVType->isStructTy())
536  report_fatal_error("A GlobalVariable of Struct type is not currently "
537  "supported by the toc data transformation.");
538 
539  assert(GVType->getPrimitiveSizeInBits() <= PointerSize * 8 &&
540  "A GlobalVariable with size larger than a TOC entry is not currently "
541  "supported by the toc data transformation.");
542 
543  if (GV->hasLocalLinkage() || GV->hasPrivateLinkage())
544  report_fatal_error("A GlobalVariable with private or local linkage is not "
545  "currently supported by the toc data transformation.");
546 
547  assert(!GV->hasCommonLinkage() &&
548  "Tentative definitions cannot have the mapping class XMC_TD.");
549 
550  return true;
551 }
552 
553 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
554 /// operand. If so Imm will receive the 32-bit value.
555 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
556  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
557  Imm = cast<ConstantSDNode>(N)->getZExtValue();
558  return true;
559  }
560  return false;
561 }
562 
563 /// isInt64Immediate - This method tests to see if the node is a 64-bit constant
564 /// operand. If so Imm will receive the 64-bit value.
566  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) {
567  Imm = cast<ConstantSDNode>(N)->getZExtValue();
568  return true;
569  }
570  return false;
571 }
572 
573 // isInt32Immediate - This method tests to see if a constant operand.
574 // If so Imm will receive the 32 bit value.
575 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
576  return isInt32Immediate(N.getNode(), Imm);
577 }
578 
579 /// isInt64Immediate - This method tests to see if the value is a 64-bit
580 /// constant operand. If so Imm will receive the 64-bit value.
582  return isInt64Immediate(N.getNode(), Imm);
583 }
584 
585 static unsigned getBranchHint(unsigned PCC,
586  const FunctionLoweringInfo &FuncInfo,
587  const SDValue &DestMBB) {
588  assert(isa<BasicBlockSDNode>(DestMBB));
589 
590  if (!FuncInfo.BPI) return PPC::BR_NO_HINT;
591 
592  const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
593  const Instruction *BBTerm = BB->getTerminator();
594 
595  if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT;
596 
597  const BasicBlock *TBB = BBTerm->getSuccessor(0);
598  const BasicBlock *FBB = BBTerm->getSuccessor(1);
599 
600  auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB);
601  auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB);
602 
603  // We only want to handle cases which are easy to predict at static time, e.g.
604  // C++ throw statement, that is very likely not taken, or calling never
605  // returned function, e.g. stdlib exit(). So we set Threshold to filter
606  // unwanted cases.
607  //
608  // Below is LLVM branch weight table, we only want to handle case 1, 2
609  //
610  // Case Taken:Nontaken Example
611  // 1. Unreachable 1048575:1 C++ throw, stdlib exit(),
612  // 2. Invoke-terminating 1:1048575
613  // 3. Coldblock 4:64 __builtin_expect
614  // 4. Loop Branch 124:4 For loop
615  // 5. PH/ZH/FPH 20:12
616  const uint32_t Threshold = 10000;
617 
618  if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb))
619  return PPC::BR_NO_HINT;
620 
621  LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName()
622  << "::" << BB->getName() << "'\n"
623  << " -> " << TBB->getName() << ": " << TProb << "\n"
624  << " -> " << FBB->getName() << ": " << FProb << "\n");
625 
626  const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB);
627 
628  // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities,
629  // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock
630  if (BBDN->getBasicBlock()->getBasicBlock() != TBB)
631  std::swap(TProb, FProb);
632 
633  return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT;
634 }
635 
636 // isOpcWithIntImmediate - This method tests to see if the node is a specific
637 // opcode and that it has a immediate integer right operand.
638 // If so Imm will receive the 32 bit value.
639 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
640  return N->getOpcode() == Opc
641  && isInt32Immediate(N->getOperand(1).getNode(), Imm);
642 }
643 
644 void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset) {
645  SDLoc dl(SN);
646  int FI = cast<FrameIndexSDNode>(N)->getIndex();
647  SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
648  unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8;
649  if (SN->hasOneUse())
650  CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI,
651  getSmallIPtrImm(Offset, dl));
652  else
653  ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI,
654  getSmallIPtrImm(Offset, dl)));
655 }
656 
657 bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask,
658  bool isShiftMask, unsigned &SH,
659  unsigned &MB, unsigned &ME) {
660  // Don't even go down this path for i64, since different logic will be
661  // necessary for rldicl/rldicr/rldimi.
662  if (N->getValueType(0) != MVT::i32)
663  return false;
664 
665  unsigned Shift = 32;
666  unsigned Indeterminant = ~0; // bit mask marking indeterminant results
667  unsigned Opcode = N->getOpcode();
668  if (N->getNumOperands() != 2 ||
669  !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31))
670  return false;
671 
672  if (Opcode == ISD::SHL) {
673  // apply shift left to mask if it comes first
674  if (isShiftMask) Mask = Mask << Shift;
675  // determine which bits are made indeterminant by shift
676  Indeterminant = ~(0xFFFFFFFFu << Shift);
677  } else if (Opcode == ISD::SRL) {
678  // apply shift right to mask if it comes first
679  if (isShiftMask) Mask = Mask >> Shift;
680  // determine which bits are made indeterminant by shift
681  Indeterminant = ~(0xFFFFFFFFu >> Shift);
682  // adjust for the left rotate
683  Shift = 32 - Shift;
684  } else if (Opcode == ISD::ROTL) {
685  Indeterminant = 0;
686  } else {
687  return false;
688  }
689 
690  // if the mask doesn't intersect any Indeterminant bits
691  if (Mask && !(Mask & Indeterminant)) {
692  SH = Shift & 31;
693  // make sure the mask is still a mask (wrap arounds may not be)
694  return isRunOfOnes(Mask, MB, ME);
695  }
696  return false;
697 }
698 
699 bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) {
700  SDValue Base = ST->getBasePtr();
701  if (Base.getOpcode() != PPCISD::ADD_TLS)
702  return false;
703  SDValue Offset = ST->getOffset();
704  if (!Offset.isUndef())
705  return false;
706  if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR)
707  return false;
708 
709  SDLoc dl(ST);
710  EVT MemVT = ST->getMemoryVT();
711  EVT RegVT = ST->getValue().getValueType();
712 
713  unsigned Opcode;
714  switch (MemVT.getSimpleVT().SimpleTy) {
715  default:
716  return false;
717  case MVT::i8: {
718  Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS;
719  break;
720  }
721  case MVT::i16: {
722  Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS;
723  break;
724  }
725  case MVT::i32: {
726  Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS;
727  break;
728  }
729  case MVT::i64: {
730  Opcode = PPC::STDXTLS;
731  break;
732  }
733  }
734  SDValue Chain = ST->getChain();
735  SDVTList VTs = ST->getVTList();
736  SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1),
737  Chain};
738  SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
739  transferMemOperands(ST, MN);
740  ReplaceNode(ST, MN);
741  return true;
742 }
743 
744 bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) {
745  SDValue Base = LD->getBasePtr();
746  if (Base.getOpcode() != PPCISD::ADD_TLS)
747  return false;
748  SDValue Offset = LD->getOffset();
749  if (!Offset.isUndef())
750  return false;
751  if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR)
752  return false;
753 
754  SDLoc dl(LD);
755  EVT MemVT = LD->getMemoryVT();
756  EVT RegVT = LD->getValueType(0);
757  unsigned Opcode;
758  switch (MemVT.getSimpleVT().SimpleTy) {
759  default:
760  return false;
761  case MVT::i8: {
762  Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS;
763  break;
764  }
765  case MVT::i16: {
766  Opcode = (RegVT == MVT::i32) ? PPC::LHZXTLS_32 : PPC::LHZXTLS;
767  break;
768  }
769  case MVT::i32: {
770  Opcode = (RegVT == MVT::i32) ? PPC::LWZXTLS_32 : PPC::LWZXTLS;
771  break;
772  }
773  case MVT::i64: {
774  Opcode = PPC::LDXTLS;
775  break;
776  }
777  }
778  SDValue Chain = LD->getChain();
779  SDVTList VTs = LD->getVTList();
780  SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain};
781  SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
782  transferMemOperands(LD, MN);
783  ReplaceNode(LD, MN);
784  return true;
785 }
786 
787 /// Turn an or of two masked values into the rotate left word immediate then
788 /// mask insert (rlwimi) instruction.
789 bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) {
790  SDValue Op0 = N->getOperand(0);
791  SDValue Op1 = N->getOperand(1);
792  SDLoc dl(N);
793 
794  KnownBits LKnown = CurDAG->computeKnownBits(Op0);
795  KnownBits RKnown = CurDAG->computeKnownBits(Op1);
796 
797  unsigned TargetMask = LKnown.Zero.getZExtValue();
798  unsigned InsertMask = RKnown.Zero.getZExtValue();
799 
800  if ((TargetMask | InsertMask) == 0xFFFFFFFF) {
801  unsigned Op0Opc = Op0.getOpcode();
802  unsigned Op1Opc = Op1.getOpcode();
803  unsigned Value, SH = 0;
804  TargetMask = ~TargetMask;
805  InsertMask = ~InsertMask;
806 
807  // If the LHS has a foldable shift and the RHS does not, then swap it to the
808  // RHS so that we can fold the shift into the insert.
809  if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) {
810  if (Op0.getOperand(0).getOpcode() == ISD::SHL ||
811  Op0.getOperand(0).getOpcode() == ISD::SRL) {
812  if (Op1.getOperand(0).getOpcode() != ISD::SHL &&
813  Op1.getOperand(0).getOpcode() != ISD::SRL) {
814  std::swap(Op0, Op1);
815  std::swap(Op0Opc, Op1Opc);
816  std::swap(TargetMask, InsertMask);
817  }
818  }
819  } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) {
820  if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL &&
821  Op1.getOperand(0).getOpcode() != ISD::SRL) {
822  std::swap(Op0, Op1);
823  std::swap(Op0Opc, Op1Opc);
824  std::swap(TargetMask, InsertMask);
825  }
826  }
827 
828  unsigned MB, ME;
829  if (isRunOfOnes(InsertMask, MB, ME)) {
830  if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) &&
831  isInt32Immediate(Op1.getOperand(1), Value)) {
832  Op1 = Op1.getOperand(0);
833  SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value;
834  }
835  if (Op1Opc == ISD::AND) {
836  // The AND mask might not be a constant, and we need to make sure that
837  // if we're going to fold the masking with the insert, all bits not
838  // know to be zero in the mask are known to be one.
839  KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1));
840  bool CanFoldMask = InsertMask == MKnown.One.getZExtValue();
841 
842  unsigned SHOpc = Op1.getOperand(0).getOpcode();
843  if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask &&
845  // Note that Value must be in range here (less than 32) because
846  // otherwise there would not be any bits set in InsertMask.
847  Op1 = Op1.getOperand(0).getOperand(0);
848  SH = (SHOpc == ISD::SHL) ? Value : 32 - Value;
849  }
850  }
851 
852  SH &= 31;
853  SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
854  getI32Imm(ME, dl) };
855  ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
856  return true;
857  }
858  }
859  return false;
860 }
861 
862 static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) {
863  unsigned MaxTruncation = 0;
864  // Cannot use range-based for loop here as we need the actual use (i.e. we
865  // need the operand number corresponding to the use). A range-based for
866  // will unbox the use and provide an SDNode*.
867  for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end();
868  Use != UseEnd; ++Use) {
869  unsigned Opc =
870  Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode();
871  switch (Opc) {
872  default: return 0;
873  case ISD::TRUNCATE:
874  if (Use->isMachineOpcode())
875  return 0;
876  MaxTruncation =
877  std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits());
878  continue;
879  case ISD::STORE: {
880  if (Use->isMachineOpcode())
881  return 0;
882  StoreSDNode *STN = cast<StoreSDNode>(*Use);
883  unsigned MemVTSize = STN->getMemoryVT().getSizeInBits();
884  if (MemVTSize == 64 || Use.getOperandNo() != 0)
885  return 0;
886  MaxTruncation = std::max(MaxTruncation, MemVTSize);
887  continue;
888  }
889  case PPC::STW8:
890  case PPC::STWX8:
891  case PPC::STWU8:
892  case PPC::STWUX8:
893  if (Use.getOperandNo() != 0)
894  return 0;
895  MaxTruncation = std::max(MaxTruncation, 32u);
896  continue;
897  case PPC::STH8:
898  case PPC::STHX8:
899  case PPC::STHU8:
900  case PPC::STHUX8:
901  if (Use.getOperandNo() != 0)
902  return 0;
903  MaxTruncation = std::max(MaxTruncation, 16u);
904  continue;
905  case PPC::STB8:
906  case PPC::STBX8:
907  case PPC::STBU8:
908  case PPC::STBUX8:
909  if (Use.getOperandNo() != 0)
910  return 0;
911  MaxTruncation = std::max(MaxTruncation, 8u);
912  continue;
913  }
914  }
915  return MaxTruncation;
916 }
917 
918 // For any 32 < Num < 64, check if the Imm contains at least Num consecutive
919 // zeros and return the number of bits by the left of these consecutive zeros.
920 static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) {
921  unsigned HiTZ = countTrailingZeros<uint32_t>(Hi_32(Imm));
922  unsigned LoLZ = countLeadingZeros<uint32_t>(Lo_32(Imm));
923  if ((HiTZ + LoLZ) >= Num)
924  return (32 + HiTZ);
925  return 0;
926 }
927 
928 // Direct materialization of 64-bit constants by enumerated patterns.
929 static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl,
930  uint64_t Imm, unsigned &InstCnt) {
931  unsigned TZ = countTrailingZeros<uint64_t>(Imm);
932  unsigned LZ = countLeadingZeros<uint64_t>(Imm);
933  unsigned TO = countTrailingOnes<uint64_t>(Imm);
934  unsigned LO = countLeadingOnes<uint64_t>(Imm);
935  unsigned Hi32 = Hi_32(Imm);
936  unsigned Lo32 = Lo_32(Imm);
937  SDNode *Result = nullptr;
938  unsigned Shift = 0;
939 
940  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
941  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
942  };
943 
944  // Following patterns use 1 instructions to materialize the Imm.
945  InstCnt = 1;
946  // 1-1) Patterns : {zeros}{15-bit valve}
947  // {ones}{15-bit valve}
948  if (isInt<16>(Imm)) {
949  SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64);
950  return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
951  }
952  // 1-2) Patterns : {zeros}{15-bit valve}{16 zeros}
953  // {ones}{15-bit valve}{16 zeros}
954  if (TZ > 15 && (LZ > 32 || LO > 32))
955  return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
956  getI32Imm((Imm >> 16) & 0xffff));
957 
958  // Following patterns use 2 instructions to materialize the Imm.
959  InstCnt = 2;
960  assert(LZ < 64 && "Unexpected leading zeros here.");
961  // Count of ones follwing the leading zeros.
962  unsigned FO = countLeadingOnes<uint64_t>(Imm << LZ);
963  // 2-1) Patterns : {zeros}{31-bit value}
964  // {ones}{31-bit value}
965  if (isInt<32>(Imm)) {
966  uint64_t ImmHi16 = (Imm >> 16) & 0xffff;
967  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
968  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
969  return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
970  getI32Imm(Imm & 0xffff));
971  }
972  // 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros}
973  // {zeros}{15-bit value}{zeros}
974  // {zeros}{ones}{15-bit value}
975  // {ones}{15-bit value}{zeros}
976  // We can take advantage of LI's sign-extension semantics to generate leading
977  // ones, and then use RLDIC to mask off the ones in both sides after rotation.
978  if ((LZ + FO + TZ) > 48) {
979  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
980  getI32Imm((Imm >> TZ) & 0xffff));
981  return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
982  getI32Imm(TZ), getI32Imm(LZ));
983  }
984  // 2-3) Pattern : {zeros}{15-bit value}{ones}
985  // Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value,
986  // therefore we can take advantage of LI's sign-extension semantics, and then
987  // mask them off after rotation.
988  //
989  // +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+
990  // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1|
991  // +------------------------+ +------------------------+
992  // 63 0 63 0
993  // Imm (Imm >> (48 - LZ) & 0xffff)
994  // +----sext-----|--16-bit--+ +clear-|-----------------+
995  // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111|
996  // +------------------------+ +------------------------+
997  // 63 0 63 0
998  // LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ
999  if ((LZ + TO) > 48) {
1000  // Since the immediates with (LZ > 32) have been handled by previous
1001  // patterns, here we have (LZ <= 32) to make sure we will not shift right
1002  // the Imm by a negative value.
1003  assert(LZ <= 32 && "Unexpected shift value.");
1004  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1005  getI32Imm((Imm >> (48 - LZ) & 0xffff)));
1006  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1007  getI32Imm(48 - LZ), getI32Imm(LZ));
1008  }
1009  // 2-4) Patterns : {zeros}{ones}{15-bit value}{ones}
1010  // {ones}{15-bit value}{ones}
1011  // We can take advantage of LI's sign-extension semantics to generate leading
1012  // ones, and then use RLDICL to mask off the ones in left sides (if required)
1013  // after rotation.
1014  //
1015  // +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+
1016  // |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb|
1017  // +------------------------+ +------------------------+
1018  // 63 0 63 0
1019  // Imm (Imm >> TO) & 0xffff
1020  // +----sext-----|--16-bit--+ +LZ|---------------------+
1021  // |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111|
1022  // +------------------------+ +------------------------+
1023  // 63 0 63 0
1024  // LI8: sext many leading zeros RLDICL: rotate left TO, clear left LZ
1025  if ((LZ + FO + TO) > 48) {
1026  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1027  getI32Imm((Imm >> TO) & 0xffff));
1028  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1029  getI32Imm(TO), getI32Imm(LZ));
1030  }
1031  // 2-5) Pattern : {32 zeros}{****}{0}{15-bit value}
1032  // If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit
1033  // value, we can use LI for Lo16 without generating leading ones then add the
1034  // Hi16(in Lo32).
1035  if (LZ == 32 && ((Lo32 & 0x8000) == 0)) {
1036  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1037  getI32Imm(Lo32 & 0xffff));
1038  return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0),
1039  getI32Imm(Lo32 >> 16));
1040  }
1041  // 2-6) Patterns : {******}{49 zeros}{******}
1042  // {******}{49 ones}{******}
1043  // If the Imm contains 49 consecutive zeros/ones, it means that a total of 15
1044  // bits remain on both sides. Rotate right the Imm to construct an int<16>
1045  // value, use LI for int<16> value and then use RLDICL without mask to rotate
1046  // it back.
1047  //
1048  // 1) findContiguousZerosAtLeast(Imm, 49)
1049  // +------|--zeros-|------+ +---ones--||---15 bit--+
1050  // |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb|
1051  // +----------------------+ +----------------------+
1052  // 63 0 63 0
1053  //
1054  // 2) findContiguousZerosAtLeast(~Imm, 49)
1055  // +------|--ones--|------+ +---ones--||---15 bit--+
1056  // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb|
1057  // +----------------------+ +----------------------+
1058  // 63 0 63 0
1059  if ((Shift = findContiguousZerosAtLeast(Imm, 49)) ||
1060  (Shift = findContiguousZerosAtLeast(~Imm, 49))) {
1061  uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1062  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1063  getI32Imm(RotImm & 0xffff));
1064  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1065  getI32Imm(Shift), getI32Imm(0));
1066  }
1067 
1068  // Following patterns use 3 instructions to materialize the Imm.
1069  InstCnt = 3;
1070  // 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros}
1071  // {zeros}{31-bit value}{zeros}
1072  // {zeros}{ones}{31-bit value}
1073  // {ones}{31-bit value}{zeros}
1074  // We can take advantage of LIS's sign-extension semantics to generate leading
1075  // ones, add the remaining bits with ORI, and then use RLDIC to mask off the
1076  // ones in both sides after rotation.
1077  if ((LZ + FO + TZ) > 32) {
1078  uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff;
1079  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1080  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1081  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1082  getI32Imm((Imm >> TZ) & 0xffff));
1083  return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
1084  getI32Imm(TZ), getI32Imm(LZ));
1085  }
1086  // 3-2) Pattern : {zeros}{31-bit value}{ones}
1087  // Shift right the Imm by (32 - LZ) bits to construct a negtive 32 bits value,
1088  // therefore we can take advantage of LIS's sign-extension semantics, add
1089  // the remaining bits with ORI, and then mask them off after rotation.
1090  // This is similar to Pattern 2-3, please refer to the diagram there.
1091  if ((LZ + TO) > 32) {
1092  // Since the immediates with (LZ > 32) have been handled by previous
1093  // patterns, here we have (LZ <= 32) to make sure we will not shift right
1094  // the Imm by a negative value.
1095  assert(LZ <= 32 && "Unexpected shift value.");
1096  Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
1097  getI32Imm((Imm >> (48 - LZ)) & 0xffff));
1098  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1099  getI32Imm((Imm >> (32 - LZ)) & 0xffff));
1100  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1101  getI32Imm(32 - LZ), getI32Imm(LZ));
1102  }
1103  // 3-3) Patterns : {zeros}{ones}{31-bit value}{ones}
1104  // {ones}{31-bit value}{ones}
1105  // We can take advantage of LIS's sign-extension semantics to generate leading
1106  // ones, add the remaining bits with ORI, and then use RLDICL to mask off the
1107  // ones in left sides (if required) after rotation.
1108  // This is similar to Pattern 2-4, please refer to the diagram there.
1109  if ((LZ + FO + TO) > 32) {
1110  Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
1111  getI32Imm((Imm >> (TO + 16)) & 0xffff));
1112  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1113  getI32Imm((Imm >> TO) & 0xffff));
1114  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1115  getI32Imm(TO), getI32Imm(LZ));
1116  }
1117  // 3-4) Patterns : High word == Low word
1118  if (Hi32 == Lo32) {
1119  // Handle the first 32 bits.
1120  uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff;
1121  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1122  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1123  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1124  getI32Imm(Lo32 & 0xffff));
1125  // Use rldimi to insert the Low word into High word.
1126  SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1127  getI32Imm(0)};
1128  return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1129  }
1130  // 3-5) Patterns : {******}{33 zeros}{******}
1131  // {******}{33 ones}{******}
1132  // If the Imm contains 33 consecutive zeros/ones, it means that a total of 31
1133  // bits remain on both sides. Rotate right the Imm to construct an int<32>
1134  // value, use LIS + ORI for int<32> value and then use RLDICL without mask to
1135  // rotate it back.
1136  // This is similar to Pattern 2-6, please refer to the diagram there.
1137  if ((Shift = findContiguousZerosAtLeast(Imm, 33)) ||
1138  (Shift = findContiguousZerosAtLeast(~Imm, 33))) {
1139  uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1140  uint64_t ImmHi16 = (RotImm >> 16) & 0xffff;
1141  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1142  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1143  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1144  getI32Imm(RotImm & 0xffff));
1145  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1146  getI32Imm(Shift), getI32Imm(0));
1147  }
1148 
1149  InstCnt = 0;
1150  return nullptr;
1151 }
1152 
1153 // Try to select instructions to generate a 64 bit immediate using prefix as
1154 // well as non prefix instructions. The function will return the SDNode
1155 // to materialize that constant or it will return nullptr if it does not
1156 // find one. The variable InstCnt is set to the number of instructions that
1157 // were selected.
1159  uint64_t Imm, unsigned &InstCnt) {
1160  unsigned TZ = countTrailingZeros<uint64_t>(Imm);
1161  unsigned LZ = countLeadingZeros<uint64_t>(Imm);
1162  unsigned TO = countTrailingOnes<uint64_t>(Imm);
1163  unsigned FO = countLeadingOnes<uint64_t>(LZ == 64 ? 0 : (Imm << LZ));
1164  unsigned Hi32 = Hi_32(Imm);
1165  unsigned Lo32 = Lo_32(Imm);
1166 
1167  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1168  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1169  };
1170 
1171  auto getI64Imm = [CurDAG, dl](uint64_t Imm) {
1172  return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
1173  };
1174 
1175  // Following patterns use 1 instruction to materialize Imm.
1176  InstCnt = 1;
1177 
1178  // The pli instruction can materialize up to 34 bits directly.
1179  // If a constant fits within 34-bits, emit the pli instruction here directly.
1180  if (isInt<34>(Imm))
1181  return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1182  CurDAG->getTargetConstant(Imm, dl, MVT::i64));
1183 
1184  // Require at least two instructions.
1185  InstCnt = 2;
1186  SDNode *Result = nullptr;
1187  // Patterns : {zeros}{ones}{33-bit value}{zeros}
1188  // {zeros}{33-bit value}{zeros}
1189  // {zeros}{ones}{33-bit value}
1190  // {ones}{33-bit value}{zeros}
1191  // We can take advantage of PLI's sign-extension semantics to generate leading
1192  // ones, and then use RLDIC to mask off the ones on both sides after rotation.
1193  if ((LZ + FO + TZ) > 30) {
1194  APInt SignedInt34 = APInt(34, (Imm >> TZ) & 0x3ffffffff);
1195  APInt Extended = SignedInt34.sext(64);
1196  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1197  getI64Imm(*Extended.getRawData()));
1198  return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
1199  getI32Imm(TZ), getI32Imm(LZ));
1200  }
1201  // Pattern : {zeros}{33-bit value}{ones}
1202  // Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value,
1203  // therefore we can take advantage of PLI's sign-extension semantics, and then
1204  // mask them off after rotation.
1205  //
1206  // +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+
1207  // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1|
1208  // +------------------------+ +------------------------+
1209  // 63 0 63 0
1210  //
1211  // +----sext-----|--34-bit--+ +clear-|-----------------+
1212  // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111|
1213  // +------------------------+ +------------------------+
1214  // 63 0 63 0
1215  if ((LZ + TO) > 30) {
1216  APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff);
1217  APInt Extended = SignedInt34.sext(64);
1218  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1219  getI64Imm(*Extended.getRawData()));
1220  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1221  getI32Imm(30 - LZ), getI32Imm(LZ));
1222  }
1223  // Patterns : {zeros}{ones}{33-bit value}{ones}
1224  // {ones}{33-bit value}{ones}
1225  // Similar to LI we can take advantage of PLI's sign-extension semantics to
1226  // generate leading ones, and then use RLDICL to mask off the ones in left
1227  // sides (if required) after rotation.
1228  if ((LZ + FO + TO) > 30) {
1229  APInt SignedInt34 = APInt(34, (Imm >> TO) & 0x3ffffffff);
1230  APInt Extended = SignedInt34.sext(64);
1231  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1232  getI64Imm(*Extended.getRawData()));
1233  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1234  getI32Imm(TO), getI32Imm(LZ));
1235  }
1236  // Patterns : {******}{31 zeros}{******}
1237  // : {******}{31 ones}{******}
1238  // If Imm contains 31 consecutive zeros/ones then the remaining bit count
1239  // is 33. Rotate right the Imm to construct a int<33> value, we can use PLI
1240  // for the int<33> value and then use RLDICL without a mask to rotate it back.
1241  //
1242  // +------|--ones--|------+ +---ones--||---33 bit--+
1243  // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb|
1244  // +----------------------+ +----------------------+
1245  // 63 0 63 0
1246  for (unsigned Shift = 0; Shift < 63; ++Shift) {
1247  uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1248  if (isInt<34>(RotImm)) {
1249  Result =
1250  CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm));
1251  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
1252  SDValue(Result, 0), getI32Imm(Shift),
1253  getI32Imm(0));
1254  }
1255  }
1256 
1257  // Patterns : High word == Low word
1258  // This is basically a splat of a 32 bit immediate.
1259  if (Hi32 == Lo32) {
1260  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32));
1261  SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1262  getI32Imm(0)};
1263  return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1264  }
1265 
1266  InstCnt = 3;
1267  // Catch-all
1268  // This pattern can form any 64 bit immediate in 3 instructions.
1269  SDNode *ResultHi =
1270  CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32));
1271  SDNode *ResultLo =
1272  CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32));
1273  SDValue Ops[] = {SDValue(ResultLo, 0), SDValue(ResultHi, 0), getI32Imm(32),
1274  getI32Imm(0)};
1275  return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1276 }
1277 
1278 static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm,
1279  unsigned *InstCnt = nullptr) {
1280  unsigned InstCntDirect = 0;
1281  // No more than 3 instructions is used if we can select the i64 immediate
1282  // directly.
1283  SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCntDirect);
1284 
1285  const PPCSubtarget &Subtarget =
1287 
1288  // If we have prefixed instructions and there is a chance we can
1289  // materialize the constant with fewer prefixed instructions than
1290  // non-prefixed, try that.
1291  if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) {
1292  unsigned InstCntDirectP = 0;
1293  SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCntDirectP);
1294  // Use the prefix case in either of two cases:
1295  // 1) We have no result from the non-prefix case to use.
1296  // 2) The non-prefix case uses more instructions than the prefix case.
1297  // If the prefix and non-prefix cases use the same number of instructions
1298  // we will prefer the non-prefix case.
1299  if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) {
1300  if (InstCnt)
1301  *InstCnt = InstCntDirectP;
1302  return ResultP;
1303  }
1304  }
1305 
1306  if (Result) {
1307  if (InstCnt)
1308  *InstCnt = InstCntDirect;
1309  return Result;
1310  }
1311  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1312  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1313  };
1314  // Handle the upper 32 bit value.
1315  Result =
1316  selectI64ImmDirect(CurDAG, dl, Imm & 0xffffffff00000000, InstCntDirect);
1317  // Add in the last bits as required.
1318  if (uint32_t Hi16 = (Lo_32(Imm) >> 16) & 0xffff) {
1319  Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64,
1320  SDValue(Result, 0), getI32Imm(Hi16));
1321  ++InstCntDirect;
1322  }
1323  if (uint32_t Lo16 = Lo_32(Imm) & 0xffff) {
1324  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1325  getI32Imm(Lo16));
1326  ++InstCntDirect;
1327  }
1328  if (InstCnt)
1329  *InstCnt = InstCntDirect;
1330  return Result;
1331 }
1332 
1333 // Select a 64-bit constant.
1335  SDLoc dl(N);
1336 
1337  // Get 64 bit value.
1338  int64_t Imm = cast<ConstantSDNode>(N)->getZExtValue();
1339  if (unsigned MinSize = allUsesTruncate(CurDAG, N)) {
1340  uint64_t SextImm = SignExtend64(Imm, MinSize);
1341  SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
1342  if (isInt<16>(SextImm))
1343  return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
1344  }
1345  return selectI64Imm(CurDAG, dl, Imm);
1346 }
1347 
1348 namespace {
1349 
1350 class BitPermutationSelector {
1351  struct ValueBit {
1352  SDValue V;
1353 
1354  // The bit number in the value, using a convention where bit 0 is the
1355  // lowest-order bit.
1356  unsigned Idx;
1357 
1358  // ConstZero means a bit we need to mask off.
1359  // Variable is a bit comes from an input variable.
1360  // VariableKnownToBeZero is also a bit comes from an input variable,
1361  // but it is known to be already zero. So we do not need to mask them.
1362  enum Kind {
1363  ConstZero,
1364  Variable,
1365  VariableKnownToBeZero
1366  } K;
1367 
1368  ValueBit(SDValue V, unsigned I, Kind K = Variable)
1369  : V(V), Idx(I), K(K) {}
1370  ValueBit(Kind K = Variable) : Idx(UINT32_MAX), K(K) {}
1371 
1372  bool isZero() const {
1373  return K == ConstZero || K == VariableKnownToBeZero;
1374  }
1375 
1376  bool hasValue() const {
1377  return K == Variable || K == VariableKnownToBeZero;
1378  }
1379 
1380  SDValue getValue() const {
1381  assert(hasValue() && "Cannot get the value of a constant bit");
1382  return V;
1383  }
1384 
1385  unsigned getValueBitIndex() const {
1386  assert(hasValue() && "Cannot get the value bit index of a constant bit");
1387  return Idx;
1388  }
1389  };
1390 
1391  // A bit group has the same underlying value and the same rotate factor.
1392  struct BitGroup {
1393  SDValue V;
1394  unsigned RLAmt;
1395  unsigned StartIdx, EndIdx;
1396 
1397  // This rotation amount assumes that the lower 32 bits of the quantity are
1398  // replicated in the high 32 bits by the rotation operator (which is done
1399  // by rlwinm and friends in 64-bit mode).
1400  bool Repl32;
1401  // Did converting to Repl32 == true change the rotation factor? If it did,
1402  // it decreased it by 32.
1403  bool Repl32CR;
1404  // Was this group coalesced after setting Repl32 to true?
1405  bool Repl32Coalesced;
1406 
1407  BitGroup(SDValue V, unsigned R, unsigned S, unsigned E)
1408  : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false),
1409  Repl32Coalesced(false) {
1410  LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R
1411  << " [" << S << ", " << E << "]\n");
1412  }
1413  };
1414 
1415  // Information on each (Value, RLAmt) pair (like the number of groups
1416  // associated with each) used to choose the lowering method.
1417  struct ValueRotInfo {
1418  SDValue V;
1419  unsigned RLAmt = std::numeric_limits<unsigned>::max();
1420  unsigned NumGroups = 0;
1421  unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max();
1422  bool Repl32 = false;
1423 
1424  ValueRotInfo() = default;
1425 
1426  // For sorting (in reverse order) by NumGroups, and then by
1427  // FirstGroupStartIdx.
1428  bool operator < (const ValueRotInfo &Other) const {
1429  // We need to sort so that the non-Repl32 come first because, when we're
1430  // doing masking, the Repl32 bit groups might be subsumed into the 64-bit
1431  // masking operation.
1432  if (Repl32 < Other.Repl32)
1433  return true;
1434  else if (Repl32 > Other.Repl32)
1435  return false;
1436  else if (NumGroups > Other.NumGroups)
1437  return true;
1438  else if (NumGroups < Other.NumGroups)
1439  return false;
1440  else if (RLAmt == 0 && Other.RLAmt != 0)
1441  return true;
1442  else if (RLAmt != 0 && Other.RLAmt == 0)
1443  return false;
1444  else if (FirstGroupStartIdx < Other.FirstGroupStartIdx)
1445  return true;
1446  return false;
1447  }
1448  };
1449 
1450  using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>;
1451  using ValueBitsMemoizer =
1453  ValueBitsMemoizer Memoizer;
1454 
1455  // Return a pair of bool and a SmallVector pointer to a memoization entry.
1456  // The bool is true if something interesting was deduced, otherwise if we're
1457  // providing only a generic representation of V (or something else likewise
1458  // uninteresting for instruction selection) through the SmallVector.
1459  std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V,
1460  unsigned NumBits) {
1461  auto &ValueEntry = Memoizer[V];
1462  if (ValueEntry)
1463  return std::make_pair(ValueEntry->first, &ValueEntry->second);
1464  ValueEntry.reset(new ValueBitsMemoizedValue());
1465  bool &Interesting = ValueEntry->first;
1466  SmallVector<ValueBit, 64> &Bits = ValueEntry->second;
1467  Bits.resize(NumBits);
1468 
1469  switch (V.getOpcode()) {
1470  default: break;
1471  case ISD::ROTL:
1472  if (isa<ConstantSDNode>(V.getOperand(1))) {
1473  unsigned RotAmt = V.getConstantOperandVal(1);
1474 
1475  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1476 
1477  for (unsigned i = 0; i < NumBits; ++i)
1478  Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt];
1479 
1480  return std::make_pair(Interesting = true, &Bits);
1481  }
1482  break;
1483  case ISD::SHL:
1484  case PPCISD::SHL:
1485  if (isa<ConstantSDNode>(V.getOperand(1))) {
1486  unsigned ShiftAmt = V.getConstantOperandVal(1);
1487 
1488  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1489 
1490  for (unsigned i = ShiftAmt; i < NumBits; ++i)
1491  Bits[i] = LHSBits[i - ShiftAmt];
1492 
1493  for (unsigned i = 0; i < ShiftAmt; ++i)
1494  Bits[i] = ValueBit(ValueBit::ConstZero);
1495 
1496  return std::make_pair(Interesting = true, &Bits);
1497  }
1498  break;
1499  case ISD::SRL:
1500  case PPCISD::SRL:
1501  if (isa<ConstantSDNode>(V.getOperand(1))) {
1502  unsigned ShiftAmt = V.getConstantOperandVal(1);
1503 
1504  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1505 
1506  for (unsigned i = 0; i < NumBits - ShiftAmt; ++i)
1507  Bits[i] = LHSBits[i + ShiftAmt];
1508 
1509  for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i)
1510  Bits[i] = ValueBit(ValueBit::ConstZero);
1511 
1512  return std::make_pair(Interesting = true, &Bits);
1513  }
1514  break;
1515  case ISD::AND:
1516  if (isa<ConstantSDNode>(V.getOperand(1))) {
1518 
1519  const SmallVector<ValueBit, 64> *LHSBits;
1520  // Mark this as interesting, only if the LHS was also interesting. This
1521  // prevents the overall procedure from matching a single immediate 'and'
1522  // (which is non-optimal because such an and might be folded with other
1523  // things if we don't select it here).
1524  std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits);
1525 
1526  for (unsigned i = 0; i < NumBits; ++i)
1527  if (((Mask >> i) & 1) == 1)
1528  Bits[i] = (*LHSBits)[i];
1529  else {
1530  // AND instruction masks this bit. If the input is already zero,
1531  // we have nothing to do here. Otherwise, make the bit ConstZero.
1532  if ((*LHSBits)[i].isZero())
1533  Bits[i] = (*LHSBits)[i];
1534  else
1535  Bits[i] = ValueBit(ValueBit::ConstZero);
1536  }
1537 
1538  return std::make_pair(Interesting, &Bits);
1539  }
1540  break;
1541  case ISD::OR: {
1542  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1543  const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second;
1544 
1545  bool AllDisjoint = true;
1546  SDValue LastVal = SDValue();
1547  unsigned LastIdx = 0;
1548  for (unsigned i = 0; i < NumBits; ++i) {
1549  if (LHSBits[i].isZero() && RHSBits[i].isZero()) {
1550  // If both inputs are known to be zero and one is ConstZero and
1551  // another is VariableKnownToBeZero, we can select whichever
1552  // we like. To minimize the number of bit groups, we select
1553  // VariableKnownToBeZero if this bit is the next bit of the same
1554  // input variable from the previous bit. Otherwise, we select
1555  // ConstZero.
1556  if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal &&
1557  LHSBits[i].getValueBitIndex() == LastIdx + 1)
1558  Bits[i] = LHSBits[i];
1559  else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal &&
1560  RHSBits[i].getValueBitIndex() == LastIdx + 1)
1561  Bits[i] = RHSBits[i];
1562  else
1563  Bits[i] = ValueBit(ValueBit::ConstZero);
1564  }
1565  else if (LHSBits[i].isZero())
1566  Bits[i] = RHSBits[i];
1567  else if (RHSBits[i].isZero())
1568  Bits[i] = LHSBits[i];
1569  else {
1570  AllDisjoint = false;
1571  break;
1572  }
1573  // We remember the value and bit index of this bit.
1574  if (Bits[i].hasValue()) {
1575  LastVal = Bits[i].getValue();
1576  LastIdx = Bits[i].getValueBitIndex();
1577  }
1578  else {
1579  if (LastVal) LastVal = SDValue();
1580  LastIdx = 0;
1581  }
1582  }
1583 
1584  if (!AllDisjoint)
1585  break;
1586 
1587  return std::make_pair(Interesting = true, &Bits);
1588  }
1589  case ISD::ZERO_EXTEND: {
1590  // We support only the case with zero extension from i32 to i64 so far.
1591  if (V.getValueType() != MVT::i64 ||
1592  V.getOperand(0).getValueType() != MVT::i32)
1593  break;
1594 
1595  const SmallVector<ValueBit, 64> *LHSBits;
1596  const unsigned NumOperandBits = 32;
1597  std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1598  NumOperandBits);
1599 
1600  for (unsigned i = 0; i < NumOperandBits; ++i)
1601  Bits[i] = (*LHSBits)[i];
1602 
1603  for (unsigned i = NumOperandBits; i < NumBits; ++i)
1604  Bits[i] = ValueBit(ValueBit::ConstZero);
1605 
1606  return std::make_pair(Interesting, &Bits);
1607  }
1608  case ISD::TRUNCATE: {
1610  EVT ToType = V.getValueType();
1611  // We support only the case with truncate from i64 to i32.
1612  if (FromType != MVT::i64 || ToType != MVT::i32)
1613  break;
1614  const unsigned NumAllBits = FromType.getSizeInBits();
1615  SmallVector<ValueBit, 64> *InBits;
1616  std::tie(Interesting, InBits) = getValueBits(V.getOperand(0),
1617  NumAllBits);
1618  const unsigned NumValidBits = ToType.getSizeInBits();
1619 
1620  // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value.
1621  // So, we cannot include this truncate.
1622  bool UseUpper32bit = false;
1623  for (unsigned i = 0; i < NumValidBits; ++i)
1624  if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) {
1625  UseUpper32bit = true;
1626  break;
1627  }
1628  if (UseUpper32bit)
1629  break;
1630 
1631  for (unsigned i = 0; i < NumValidBits; ++i)
1632  Bits[i] = (*InBits)[i];
1633 
1634  return std::make_pair(Interesting, &Bits);
1635  }
1636  case ISD::AssertZext: {
1637  // For AssertZext, we look through the operand and
1638  // mark the bits known to be zero.
1639  const SmallVector<ValueBit, 64> *LHSBits;
1640  std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1641  NumBits);
1642 
1643  EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT();
1644  const unsigned NumValidBits = FromType.getSizeInBits();
1645  for (unsigned i = 0; i < NumValidBits; ++i)
1646  Bits[i] = (*LHSBits)[i];
1647 
1648  // These bits are known to be zero but the AssertZext may be from a value
1649  // that already has some constant zero bits (i.e. from a masking and).
1650  for (unsigned i = NumValidBits; i < NumBits; ++i)
1651  Bits[i] = (*LHSBits)[i].hasValue()
1652  ? ValueBit((*LHSBits)[i].getValue(),
1653  (*LHSBits)[i].getValueBitIndex(),
1654  ValueBit::VariableKnownToBeZero)
1655  : ValueBit(ValueBit::ConstZero);
1656 
1657  return std::make_pair(Interesting, &Bits);
1658  }
1659  case ISD::LOAD:
1660  LoadSDNode *LD = cast<LoadSDNode>(V);
1661  if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) {
1662  EVT VT = LD->getMemoryVT();
1663  const unsigned NumValidBits = VT.getSizeInBits();
1664 
1665  for (unsigned i = 0; i < NumValidBits; ++i)
1666  Bits[i] = ValueBit(V, i);
1667 
1668  // These bits are known to be zero.
1669  for (unsigned i = NumValidBits; i < NumBits; ++i)
1670  Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero);
1671 
1672  // Zero-extending load itself cannot be optimized. So, it is not
1673  // interesting by itself though it gives useful information.
1674  return std::make_pair(Interesting = false, &Bits);
1675  }
1676  break;
1677  }
1678 
1679  for (unsigned i = 0; i < NumBits; ++i)
1680  Bits[i] = ValueBit(V, i);
1681 
1682  return std::make_pair(Interesting = false, &Bits);
1683  }
1684 
1685  // For each value (except the constant ones), compute the left-rotate amount
1686  // to get it from its original to final position.
1687  void computeRotationAmounts() {
1688  NeedMask = false;
1689  RLAmt.resize(Bits.size());
1690  for (unsigned i = 0; i < Bits.size(); ++i)
1691  if (Bits[i].hasValue()) {
1692  unsigned VBI = Bits[i].getValueBitIndex();
1693  if (i >= VBI)
1694  RLAmt[i] = i - VBI;
1695  else
1696  RLAmt[i] = Bits.size() - (VBI - i);
1697  } else if (Bits[i].isZero()) {
1698  NeedMask = true;
1699  RLAmt[i] = UINT32_MAX;
1700  } else {
1701  llvm_unreachable("Unknown value bit type");
1702  }
1703  }
1704 
1705  // Collect groups of consecutive bits with the same underlying value and
1706  // rotation factor. If we're doing late masking, we ignore zeros, otherwise
1707  // they break up groups.
1708  void collectBitGroups(bool LateMask) {
1709  BitGroups.clear();
1710 
1711  unsigned LastRLAmt = RLAmt[0];
1712  SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue();
1713  unsigned LastGroupStartIdx = 0;
1714  bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1715  for (unsigned i = 1; i < Bits.size(); ++i) {
1716  unsigned ThisRLAmt = RLAmt[i];
1717  SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue();
1718  if (LateMask && !ThisValue) {
1719  ThisValue = LastValue;
1720  ThisRLAmt = LastRLAmt;
1721  // If we're doing late masking, then the first bit group always starts
1722  // at zero (even if the first bits were zero).
1723  if (BitGroups.empty())
1724  LastGroupStartIdx = 0;
1725  }
1726 
1727  // If this bit is known to be zero and the current group is a bit group
1728  // of zeros, we do not need to terminate the current bit group even the
1729  // Value or RLAmt does not match here. Instead, we terminate this group
1730  // when the first non-zero bit appears later.
1731  if (IsGroupOfZeros && Bits[i].isZero())
1732  continue;
1733 
1734  // If this bit has the same underlying value and the same rotate factor as
1735  // the last one, then they're part of the same group.
1736  if (ThisRLAmt == LastRLAmt && ThisValue == LastValue)
1737  // We cannot continue the current group if this bits is not known to
1738  // be zero in a bit group of zeros.
1739  if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero()))
1740  continue;
1741 
1742  if (LastValue.getNode())
1743  BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1744  i-1));
1745  LastRLAmt = ThisRLAmt;
1746  LastValue = ThisValue;
1747  LastGroupStartIdx = i;
1748  IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1749  }
1750  if (LastValue.getNode())
1751  BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1752  Bits.size()-1));
1753 
1754  if (BitGroups.empty())
1755  return;
1756 
1757  // We might be able to combine the first and last groups.
1758  if (BitGroups.size() > 1) {
1759  // If the first and last groups are the same, then remove the first group
1760  // in favor of the last group, making the ending index of the last group
1761  // equal to the ending index of the to-be-removed first group.
1762  if (BitGroups[0].StartIdx == 0 &&
1763  BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 &&
1764  BitGroups[0].V == BitGroups[BitGroups.size()-1].V &&
1765  BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) {
1766  LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n");
1767  BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx;
1768  BitGroups.erase(BitGroups.begin());
1769  }
1770  }
1771  }
1772 
1773  // Take all (SDValue, RLAmt) pairs and sort them by the number of groups
1774  // associated with each. If the number of groups are same, we prefer a group
1775  // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate
1776  // instruction. If there is a degeneracy, pick the one that occurs
1777  // first (in the final value).
1778  void collectValueRotInfo() {
1779  ValueRots.clear();
1780 
1781  for (auto &BG : BitGroups) {
1782  unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0);
1783  ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)];
1784  VRI.V = BG.V;
1785  VRI.RLAmt = BG.RLAmt;
1786  VRI.Repl32 = BG.Repl32;
1787  VRI.NumGroups += 1;
1788  VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx);
1789  }
1790 
1791  // Now that we've collected the various ValueRotInfo instances, we need to
1792  // sort them.
1793  ValueRotsVec.clear();
1794  for (auto &I : ValueRots) {
1795  ValueRotsVec.push_back(I.second);
1796  }
1797  llvm::sort(ValueRotsVec);
1798  }
1799 
1800  // In 64-bit mode, rlwinm and friends have a rotation operator that
1801  // replicates the low-order 32 bits into the high-order 32-bits. The mask
1802  // indices of these instructions can only be in the lower 32 bits, so they
1803  // can only represent some 64-bit bit groups. However, when they can be used,
1804  // the 32-bit replication can be used to represent, as a single bit group,
1805  // otherwise separate bit groups. We'll convert to replicated-32-bit bit
1806  // groups when possible. Returns true if any of the bit groups were
1807  // converted.
1808  void assignRepl32BitGroups() {
1809  // If we have bits like this:
1810  //
1811  // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
1812  // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24
1813  // Groups: | RLAmt = 8 | RLAmt = 40 |
1814  //
1815  // But, making use of a 32-bit operation that replicates the low-order 32
1816  // bits into the high-order 32 bits, this can be one bit group with a RLAmt
1817  // of 8.
1818 
1819  auto IsAllLow32 = [this](BitGroup & BG) {
1820  if (BG.StartIdx <= BG.EndIdx) {
1821  for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) {
1822  if (!Bits[i].hasValue())
1823  continue;
1824  if (Bits[i].getValueBitIndex() >= 32)
1825  return false;
1826  }
1827  } else {
1828  for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) {
1829  if (!Bits[i].hasValue())
1830  continue;
1831  if (Bits[i].getValueBitIndex() >= 32)
1832  return false;
1833  }
1834  for (unsigned i = 0; i <= BG.EndIdx; ++i) {
1835  if (!Bits[i].hasValue())
1836  continue;
1837  if (Bits[i].getValueBitIndex() >= 32)
1838  return false;
1839  }
1840  }
1841 
1842  return true;
1843  };
1844 
1845  for (auto &BG : BitGroups) {
1846  // If this bit group has RLAmt of 0 and will not be merged with
1847  // another bit group, we don't benefit from Repl32. We don't mark
1848  // such group to give more freedom for later instruction selection.
1849  if (BG.RLAmt == 0) {
1850  auto PotentiallyMerged = [this](BitGroup & BG) {
1851  for (auto &BG2 : BitGroups)
1852  if (&BG != &BG2 && BG.V == BG2.V &&
1853  (BG2.RLAmt == 0 || BG2.RLAmt == 32))
1854  return true;
1855  return false;
1856  };
1857  if (!PotentiallyMerged(BG))
1858  continue;
1859  }
1860  if (BG.StartIdx < 32 && BG.EndIdx < 32) {
1861  if (IsAllLow32(BG)) {
1862  if (BG.RLAmt >= 32) {
1863  BG.RLAmt -= 32;
1864  BG.Repl32CR = true;
1865  }
1866 
1867  BG.Repl32 = true;
1868 
1869  LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for "
1870  << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " ["
1871  << BG.StartIdx << ", " << BG.EndIdx << "]\n");
1872  }
1873  }
1874  }
1875 
1876  // Now walk through the bit groups, consolidating where possible.
1877  for (auto I = BitGroups.begin(); I != BitGroups.end();) {
1878  // We might want to remove this bit group by merging it with the previous
1879  // group (which might be the ending group).
1880  auto IP = (I == BitGroups.begin()) ?
1881  std::prev(BitGroups.end()) : std::prev(I);
1882  if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt &&
1883  I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) {
1884 
1885  LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for "
1886  << I->V.getNode() << " RLAmt = " << I->RLAmt << " ["
1887  << I->StartIdx << ", " << I->EndIdx
1888  << "] with group with range [" << IP->StartIdx << ", "
1889  << IP->EndIdx << "]\n");
1890 
1891  IP->EndIdx = I->EndIdx;
1892  IP->Repl32CR = IP->Repl32CR || I->Repl32CR;
1893  IP->Repl32Coalesced = true;
1894  I = BitGroups.erase(I);
1895  continue;
1896  } else {
1897  // There is a special case worth handling: If there is a single group
1898  // covering the entire upper 32 bits, and it can be merged with both
1899  // the next and previous groups (which might be the same group), then
1900  // do so. If it is the same group (so there will be only one group in
1901  // total), then we need to reverse the order of the range so that it
1902  // covers the entire 64 bits.
1903  if (I->StartIdx == 32 && I->EndIdx == 63) {
1904  assert(std::next(I) == BitGroups.end() &&
1905  "bit group ends at index 63 but there is another?");
1906  auto IN = BitGroups.begin();
1907 
1908  if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V &&
1909  (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt &&
1910  IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP &&
1911  IsAllLow32(*I)) {
1912 
1913  LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode()
1914  << " RLAmt = " << I->RLAmt << " [" << I->StartIdx
1915  << ", " << I->EndIdx
1916  << "] with 32-bit replicated groups with ranges ["
1917  << IP->StartIdx << ", " << IP->EndIdx << "] and ["
1918  << IN->StartIdx << ", " << IN->EndIdx << "]\n");
1919 
1920  if (IP == IN) {
1921  // There is only one other group; change it to cover the whole
1922  // range (backward, so that it can still be Repl32 but cover the
1923  // whole 64-bit range).
1924  IP->StartIdx = 31;
1925  IP->EndIdx = 30;
1926  IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32;
1927  IP->Repl32Coalesced = true;
1928  I = BitGroups.erase(I);
1929  } else {
1930  // There are two separate groups, one before this group and one
1931  // after us (at the beginning). We're going to remove this group,
1932  // but also the group at the very beginning.
1933  IP->EndIdx = IN->EndIdx;
1934  IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32;
1935  IP->Repl32Coalesced = true;
1936  I = BitGroups.erase(I);
1937  BitGroups.erase(BitGroups.begin());
1938  }
1939 
1940  // This must be the last group in the vector (and we might have
1941  // just invalidated the iterator above), so break here.
1942  break;
1943  }
1944  }
1945  }
1946 
1947  ++I;
1948  }
1949  }
1950 
1951  SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
1952  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1953  }
1954 
1955  uint64_t getZerosMask() {
1956  uint64_t Mask = 0;
1957  for (unsigned i = 0; i < Bits.size(); ++i) {
1958  if (Bits[i].hasValue())
1959  continue;
1960  Mask |= (UINT64_C(1) << i);
1961  }
1962 
1963  return ~Mask;
1964  }
1965 
1966  // This method extends an input value to 64 bit if input is 32-bit integer.
1967  // While selecting instructions in BitPermutationSelector in 64-bit mode,
1968  // an input value can be a 32-bit integer if a ZERO_EXTEND node is included.
1969  // In such case, we extend it to 64 bit to be consistent with other values.
1970  SDValue ExtendToInt64(SDValue V, const SDLoc &dl) {
1971  if (V.getValueSizeInBits() == 64)
1972  return V;
1973 
1974  assert(V.getValueSizeInBits() == 32);
1975  SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1976  SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
1977  MVT::i64), 0);
1978  SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
1979  MVT::i64, ImDef, V,
1980  SubRegIdx), 0);
1981  return ExtVal;
1982  }
1983 
1984  SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
1985  if (V.getValueSizeInBits() == 32)
1986  return V;
1987 
1988  assert(V.getValueSizeInBits() == 64);
1989  SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1990  SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
1991  MVT::i32, V, SubRegIdx), 0);
1992  return SubVal;
1993  }
1994 
1995  // Depending on the number of groups for a particular value, it might be
1996  // better to rotate, mask explicitly (using andi/andis), and then or the
1997  // result. Select this part of the result first.
1998  void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2000  return;
2001 
2002  for (ValueRotInfo &VRI : ValueRotsVec) {
2003  unsigned Mask = 0;
2004  for (unsigned i = 0; i < Bits.size(); ++i) {
2005  if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V)
2006  continue;
2007  if (RLAmt[i] != VRI.RLAmt)
2008  continue;
2009  Mask |= (1u << i);
2010  }
2011 
2012  // Compute the masks for andi/andis that would be necessary.
2013  unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2014  assert((ANDIMask != 0 || ANDISMask != 0) &&
2015  "No set bits in mask for value bit groups");
2016  bool NeedsRotate = VRI.RLAmt != 0;
2017 
2018  // We're trying to minimize the number of instructions. If we have one
2019  // group, using one of andi/andis can break even. If we have three
2020  // groups, we can use both andi and andis and break even (to use both
2021  // andi and andis we also need to or the results together). We need four
2022  // groups if we also need to rotate. To use andi/andis we need to do more
2023  // than break even because rotate-and-mask instructions tend to be easier
2024  // to schedule.
2025 
2026  // FIXME: We've biased here against using andi/andis, which is right for
2027  // POWER cores, but not optimal everywhere. For example, on the A2,
2028  // andi/andis have single-cycle latency whereas the rotate-and-mask
2029  // instructions take two cycles, and it would be better to bias toward
2030  // andi/andis in break-even cases.
2031 
2032  unsigned NumAndInsts = (unsigned) NeedsRotate +
2033  (unsigned) (ANDIMask != 0) +
2034  (unsigned) (ANDISMask != 0) +
2035  (unsigned) (ANDIMask != 0 && ANDISMask != 0) +
2036  (unsigned) (bool) Res;
2037 
2038  LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
2039  << " RL: " << VRI.RLAmt << ":"
2040  << "\n\t\t\tisel using masking: " << NumAndInsts
2041  << " using rotates: " << VRI.NumGroups << "\n");
2042 
2043  if (NumAndInsts >= VRI.NumGroups)
2044  continue;
2045 
2046  LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
2047 
2048  if (InstCnt) *InstCnt += NumAndInsts;
2049 
2050  SDValue VRot;
2051  if (VRI.RLAmt) {
2052  SDValue Ops[] =
2053  { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2054  getI32Imm(0, dl), getI32Imm(31, dl) };
2055  VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2056  Ops), 0);
2057  } else {
2058  VRot = TruncateToInt32(VRI.V, dl);
2059  }
2060 
2061  SDValue ANDIVal, ANDISVal;
2062  if (ANDIMask != 0)
2063  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
2064  VRot, getI32Imm(ANDIMask, dl)),
2065  0);
2066  if (ANDISMask != 0)
2067  ANDISVal =
2068  SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot,
2069  getI32Imm(ANDISMask, dl)),
2070  0);
2071 
2072  SDValue TotalVal;
2073  if (!ANDIVal)
2074  TotalVal = ANDISVal;
2075  else if (!ANDISVal)
2076  TotalVal = ANDIVal;
2077  else
2078  TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2079  ANDIVal, ANDISVal), 0);
2080 
2081  if (!Res)
2082  Res = TotalVal;
2083  else
2084  Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2085  Res, TotalVal), 0);
2086 
2087  // Now, remove all groups with this underlying value and rotation
2088  // factor.
2089  eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2090  return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2091  });
2092  }
2093  }
2094 
2095  // Instruction selection for the 32-bit case.
2096  SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) {
2097  SDLoc dl(N);
2098  SDValue Res;
2099 
2100  if (InstCnt) *InstCnt = 0;
2101 
2102  // Take care of cases that should use andi/andis first.
2103  SelectAndParts32(dl, Res, InstCnt);
2104 
2105  // If we've not yet selected a 'starting' instruction, and we have no zeros
2106  // to fill in, select the (Value, RLAmt) with the highest priority (largest
2107  // number of groups), and start with this rotated value.
2108  if ((!NeedMask || LateMask) && !Res) {
2109  ValueRotInfo &VRI = ValueRotsVec[0];
2110  if (VRI.RLAmt) {
2111  if (InstCnt) *InstCnt += 1;
2112  SDValue Ops[] =
2113  { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2114  getI32Imm(0, dl), getI32Imm(31, dl) };
2115  Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
2116  0);
2117  } else {
2118  Res = TruncateToInt32(VRI.V, dl);
2119  }
2120 
2121  // Now, remove all groups with this underlying value and rotation factor.
2122  eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2123  return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2124  });
2125  }
2126 
2127  if (InstCnt) *InstCnt += BitGroups.size();
2128 
2129  // Insert the other groups (one at a time).
2130  for (auto &BG : BitGroups) {
2131  if (!Res) {
2132  SDValue Ops[] =
2133  { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2134  getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
2135  getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
2136  Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
2137  } else {
2138  SDValue Ops[] =
2139  { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2140  getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
2141  getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
2142  Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0);
2143  }
2144  }
2145 
2146  if (LateMask) {
2147  unsigned Mask = (unsigned) getZerosMask();
2148 
2149  unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2150  assert((ANDIMask != 0 || ANDISMask != 0) &&
2151  "No set bits in zeros mask?");
2152 
2153  if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2154  (unsigned) (ANDISMask != 0) +
2155  (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2156 
2157  SDValue ANDIVal, ANDISVal;
2158  if (ANDIMask != 0)
2159  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
2160  Res, getI32Imm(ANDIMask, dl)),
2161  0);
2162  if (ANDISMask != 0)
2163  ANDISVal =
2164  SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res,
2165  getI32Imm(ANDISMask, dl)),
2166  0);
2167 
2168  if (!ANDIVal)
2169  Res = ANDISVal;
2170  else if (!ANDISVal)
2171  Res = ANDIVal;
2172  else
2173  Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2174  ANDIVal, ANDISVal), 0);
2175  }
2176 
2177  return Res.getNode();
2178  }
2179 
2180  unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32,
2181  unsigned MaskStart, unsigned MaskEnd,
2182  bool IsIns) {
2183  // In the notation used by the instructions, 'start' and 'end' are reversed
2184  // because bits are counted from high to low order.
2185  unsigned InstMaskStart = 64 - MaskEnd - 1,
2186  InstMaskEnd = 64 - MaskStart - 1;
2187 
2188  if (Repl32)
2189  return 1;
2190 
2191  if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) ||
2192  InstMaskEnd == 63 - RLAmt)
2193  return 1;
2194 
2195  return 2;
2196  }
2197 
2198  // For 64-bit values, not all combinations of rotates and masks are
2199  // available. Produce one if it is available.
2200  SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt,
2201  bool Repl32, unsigned MaskStart, unsigned MaskEnd,
2202  unsigned *InstCnt = nullptr) {
2203  // In the notation used by the instructions, 'start' and 'end' are reversed
2204  // because bits are counted from high to low order.
2205  unsigned InstMaskStart = 64 - MaskEnd - 1,
2206  InstMaskEnd = 64 - MaskStart - 1;
2207 
2208  if (InstCnt) *InstCnt += 1;
2209 
2210  if (Repl32) {
2211  // This rotation amount assumes that the lower 32 bits of the quantity
2212  // are replicated in the high 32 bits by the rotation operator (which is
2213  // done by rlwinm and friends).
2214  assert(InstMaskStart >= 32 && "Mask cannot start out of range");
2215  assert(InstMaskEnd >= 32 && "Mask cannot end out of range");
2216  SDValue Ops[] =
2217  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2218  getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2219  return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64,
2220  Ops), 0);
2221  }
2222 
2223  if (InstMaskEnd == 63) {
2224  SDValue Ops[] =
2225  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2226  getI32Imm(InstMaskStart, dl) };
2227  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0);
2228  }
2229 
2230  if (InstMaskStart == 0) {
2231  SDValue Ops[] =
2232  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2233  getI32Imm(InstMaskEnd, dl) };
2234  return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0);
2235  }
2236 
2237  if (InstMaskEnd == 63 - RLAmt) {
2238  SDValue Ops[] =
2239  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2240  getI32Imm(InstMaskStart, dl) };
2241  return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0);
2242  }
2243 
2244  // We cannot do this with a single instruction, so we'll use two. The
2245  // problem is that we're not free to choose both a rotation amount and mask
2246  // start and end independently. We can choose an arbitrary mask start and
2247  // end, but then the rotation amount is fixed. Rotation, however, can be
2248  // inverted, and so by applying an "inverse" rotation first, we can get the
2249  // desired result.
2250  if (InstCnt) *InstCnt += 1;
2251 
2252  // The rotation mask for the second instruction must be MaskStart.
2253  unsigned RLAmt2 = MaskStart;
2254  // The first instruction must rotate V so that the overall rotation amount
2255  // is RLAmt.
2256  unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2257  if (RLAmt1)
2258  V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2259  return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd);
2260  }
2261 
2262  // For 64-bit values, not all combinations of rotates and masks are
2263  // available. Produce a rotate-mask-and-insert if one is available.
2264  SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
2265  unsigned RLAmt, bool Repl32, unsigned MaskStart,
2266  unsigned MaskEnd, unsigned *InstCnt = nullptr) {
2267  // In the notation used by the instructions, 'start' and 'end' are reversed
2268  // because bits are counted from high to low order.
2269  unsigned InstMaskStart = 64 - MaskEnd - 1,
2270  InstMaskEnd = 64 - MaskStart - 1;
2271 
2272  if (InstCnt) *InstCnt += 1;
2273 
2274  if (Repl32) {
2275  // This rotation amount assumes that the lower 32 bits of the quantity
2276  // are replicated in the high 32 bits by the rotation operator (which is
2277  // done by rlwinm and friends).
2278  assert(InstMaskStart >= 32 && "Mask cannot start out of range");
2279  assert(InstMaskEnd >= 32 && "Mask cannot end out of range");
2280  SDValue Ops[] =
2281  { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2282  getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2283  return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64,
2284  Ops), 0);
2285  }
2286 
2287  if (InstMaskEnd == 63 - RLAmt) {
2288  SDValue Ops[] =
2289  { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2290  getI32Imm(InstMaskStart, dl) };
2291  return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0);
2292  }
2293 
2294  // We cannot do this with a single instruction, so we'll use two. The
2295  // problem is that we're not free to choose both a rotation amount and mask
2296  // start and end independently. We can choose an arbitrary mask start and
2297  // end, but then the rotation amount is fixed. Rotation, however, can be
2298  // inverted, and so by applying an "inverse" rotation first, we can get the
2299  // desired result.
2300  if (InstCnt) *InstCnt += 1;
2301 
2302  // The rotation mask for the second instruction must be MaskStart.
2303  unsigned RLAmt2 = MaskStart;
2304  // The first instruction must rotate V so that the overall rotation amount
2305  // is RLAmt.
2306  unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2307  if (RLAmt1)
2308  V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2309  return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd);
2310  }
2311 
2312  void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2314  return;
2315 
2316  // The idea here is the same as in the 32-bit version, but with additional
2317  // complications from the fact that Repl32 might be true. Because we
2318  // aggressively convert bit groups to Repl32 form (which, for small
2319  // rotation factors, involves no other change), and then coalesce, it might
2320  // be the case that a single 64-bit masking operation could handle both
2321  // some Repl32 groups and some non-Repl32 groups. If converting to Repl32
2322  // form allowed coalescing, then we must use a 32-bit rotaton in order to
2323  // completely capture the new combined bit group.
2324 
2325  for (ValueRotInfo &VRI : ValueRotsVec) {
2326  uint64_t Mask = 0;
2327 
2328  // We need to add to the mask all bits from the associated bit groups.
2329  // If Repl32 is false, we need to add bits from bit groups that have
2330  // Repl32 true, but are trivially convertable to Repl32 false. Such a
2331  // group is trivially convertable if it overlaps only with the lower 32
2332  // bits, and the group has not been coalesced.
2333  auto MatchingBG = [VRI](const BitGroup &BG) {
2334  if (VRI.V != BG.V)
2335  return false;
2336 
2337  unsigned EffRLAmt = BG.RLAmt;
2338  if (!VRI.Repl32 && BG.Repl32) {
2339  if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx &&
2340  !BG.Repl32Coalesced) {
2341  if (BG.Repl32CR)
2342  EffRLAmt += 32;
2343  } else {
2344  return false;
2345  }
2346  } else if (VRI.Repl32 != BG.Repl32) {
2347  return false;
2348  }
2349 
2350  return VRI.RLAmt == EffRLAmt;
2351  };
2352 
2353  for (auto &BG : BitGroups) {
2354  if (!MatchingBG(BG))
2355  continue;
2356 
2357  if (BG.StartIdx <= BG.EndIdx) {
2358  for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i)
2359  Mask |= (UINT64_C(1) << i);
2360  } else {
2361  for (unsigned i = BG.StartIdx; i < Bits.size(); ++i)
2362  Mask |= (UINT64_C(1) << i);
2363  for (unsigned i = 0; i <= BG.EndIdx; ++i)
2364  Mask |= (UINT64_C(1) << i);
2365  }
2366  }
2367 
2368  // We can use the 32-bit andi/andis technique if the mask does not
2369  // require any higher-order bits. This can save an instruction compared
2370  // to always using the general 64-bit technique.
2371  bool Use32BitInsts = isUInt<32>(Mask);
2372  // Compute the masks for andi/andis that would be necessary.
2373  unsigned ANDIMask = (Mask & UINT16_MAX),
2374  ANDISMask = (Mask >> 16) & UINT16_MAX;
2375 
2376  bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask));
2377 
2378  unsigned NumAndInsts = (unsigned) NeedsRotate +
2379  (unsigned) (bool) Res;
2380  unsigned NumOfSelectInsts = 0;
2381  selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts);
2382  assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant.");
2383  if (Use32BitInsts)
2384  NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) +
2385  (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2386  else
2387  NumAndInsts += NumOfSelectInsts + /* and */ 1;
2388 
2389  unsigned NumRLInsts = 0;
2390  bool FirstBG = true;
2391  bool MoreBG = false;
2392  for (auto &BG : BitGroups) {
2393  if (!MatchingBG(BG)) {
2394  MoreBG = true;
2395  continue;
2396  }
2397  NumRLInsts +=
2398  SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx,
2399  !FirstBG);
2400  FirstBG = false;
2401  }
2402 
2403  LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
2404  << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":")
2405  << "\n\t\t\tisel using masking: " << NumAndInsts
2406  << " using rotates: " << NumRLInsts << "\n");
2407 
2408  // When we'd use andi/andis, we bias toward using the rotates (andi only
2409  // has a record form, and is cracked on POWER cores). However, when using
2410  // general 64-bit constant formation, bias toward the constant form,
2411  // because that exposes more opportunities for CSE.
2412  if (NumAndInsts > NumRLInsts)
2413  continue;
2414  // When merging multiple bit groups, instruction or is used.
2415  // But when rotate is used, rldimi can inert the rotated value into any
2416  // register, so instruction or can be avoided.
2417  if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts)
2418  continue;
2419 
2420  LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
2421 
2422  if (InstCnt) *InstCnt += NumAndInsts;
2423 
2424  SDValue VRot;
2425  // We actually need to generate a rotation if we have a non-zero rotation
2426  // factor or, in the Repl32 case, if we care about any of the
2427  // higher-order replicated bits. In the latter case, we generate a mask
2428  // backward so that it actually includes the entire 64 bits.
2429  if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)))
2430  VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2431  VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63);
2432  else
2433  VRot = VRI.V;
2434 
2435  SDValue TotalVal;
2436  if (Use32BitInsts) {
2437  assert((ANDIMask != 0 || ANDISMask != 0) &&
2438  "No set bits in mask when using 32-bit ands for 64-bit value");
2439 
2440  SDValue ANDIVal, ANDISVal;
2441  if (ANDIMask != 0)
2442  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2443  ExtendToInt64(VRot, dl),
2444  getI32Imm(ANDIMask, dl)),
2445  0);
2446  if (ANDISMask != 0)
2447  ANDISVal =
2448  SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2449  ExtendToInt64(VRot, dl),
2450  getI32Imm(ANDISMask, dl)),
2451  0);
2452 
2453  if (!ANDIVal)
2454  TotalVal = ANDISVal;
2455  else if (!ANDISVal)
2456  TotalVal = ANDIVal;
2457  else
2458  TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2459  ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2460  } else {
2461  TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0);
2462  TotalVal =
2463  SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2464  ExtendToInt64(VRot, dl), TotalVal),
2465  0);
2466  }
2467 
2468  if (!Res)
2469  Res = TotalVal;
2470  else
2471  Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2472  ExtendToInt64(Res, dl), TotalVal),
2473  0);
2474 
2475  // Now, remove all groups with this underlying value and rotation
2476  // factor.
2477  eraseMatchingBitGroups(MatchingBG);
2478  }
2479  }
2480 
2481  // Instruction selection for the 64-bit case.
2482  SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) {
2483  SDLoc dl(N);
2484  SDValue Res;
2485 
2486  if (InstCnt) *InstCnt = 0;
2487 
2488  // Take care of cases that should use andi/andis first.
2489  SelectAndParts64(dl, Res, InstCnt);
2490 
2491  // If we've not yet selected a 'starting' instruction, and we have no zeros
2492  // to fill in, select the (Value, RLAmt) with the highest priority (largest
2493  // number of groups), and start with this rotated value.
2494  if ((!NeedMask || LateMask) && !Res) {
2495  // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32
2496  // groups will come first, and so the VRI representing the largest number
2497  // of groups might not be first (it might be the first Repl32 groups).
2498  unsigned MaxGroupsIdx = 0;
2499  if (!ValueRotsVec[0].Repl32) {
2500  for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i)
2501  if (ValueRotsVec[i].Repl32) {
2502  if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups)
2503  MaxGroupsIdx = i;
2504  break;
2505  }
2506  }
2507 
2508  ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx];
2509  bool NeedsRotate = false;
2510  if (VRI.RLAmt) {
2511  NeedsRotate = true;
2512  } else if (VRI.Repl32) {
2513  for (auto &BG : BitGroups) {
2514  if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt ||
2515  BG.Repl32 != VRI.Repl32)
2516  continue;
2517 
2518  // We don't need a rotate if the bit group is confined to the lower
2519  // 32 bits.
2520  if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx)
2521  continue;
2522 
2523  NeedsRotate = true;
2524  break;
2525  }
2526  }
2527 
2528  if (NeedsRotate)
2529  Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2530  VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63,
2531  InstCnt);
2532  else
2533  Res = VRI.V;
2534 
2535  // Now, remove all groups with this underlying value and rotation factor.
2536  if (Res)
2537  eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2538  return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt &&
2539  BG.Repl32 == VRI.Repl32;
2540  });
2541  }
2542 
2543  // Because 64-bit rotates are more flexible than inserts, we might have a
2544  // preference regarding which one we do first (to save one instruction).
2545  if (!Res)
2546  for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) {
2547  if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2548  false) <
2549  SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2550  true)) {
2551  if (I != BitGroups.begin()) {
2552  BitGroup BG = *I;
2553  BitGroups.erase(I);
2554  BitGroups.insert(BitGroups.begin(), BG);
2555  }
2556 
2557  break;
2558  }
2559  }
2560 
2561  // Insert the other groups (one at a time).
2562  for (auto &BG : BitGroups) {
2563  if (!Res)
2564  Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx,
2565  BG.EndIdx, InstCnt);
2566  else
2567  Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32,
2568  BG.StartIdx, BG.EndIdx, InstCnt);
2569  }
2570 
2571  if (LateMask) {
2572  uint64_t Mask = getZerosMask();
2573 
2574  // We can use the 32-bit andi/andis technique if the mask does not
2575  // require any higher-order bits. This can save an instruction compared
2576  // to always using the general 64-bit technique.
2577  bool Use32BitInsts = isUInt<32>(Mask);
2578  // Compute the masks for andi/andis that would be necessary.
2579  unsigned ANDIMask = (Mask & UINT16_MAX),
2580  ANDISMask = (Mask >> 16) & UINT16_MAX;
2581 
2582  if (Use32BitInsts) {
2583  assert((ANDIMask != 0 || ANDISMask != 0) &&
2584  "No set bits in mask when using 32-bit ands for 64-bit value");
2585 
2586  if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2587  (unsigned) (ANDISMask != 0) +
2588  (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2589 
2590  SDValue ANDIVal, ANDISVal;
2591  if (ANDIMask != 0)
2592  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2593  ExtendToInt64(Res, dl),
2594  getI32Imm(ANDIMask, dl)),
2595  0);
2596  if (ANDISMask != 0)
2597  ANDISVal =
2598  SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2599  ExtendToInt64(Res, dl),
2600  getI32Imm(ANDISMask, dl)),
2601  0);
2602 
2603  if (!ANDIVal)
2604  Res = ANDISVal;
2605  else if (!ANDISVal)
2606  Res = ANDIVal;
2607  else
2608  Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2609  ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2610  } else {
2611  unsigned NumOfSelectInsts = 0;
2612  SDValue MaskVal =
2613  SDValue(selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts), 0);
2614  Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2615  ExtendToInt64(Res, dl), MaskVal),
2616  0);
2617  if (InstCnt)
2618  *InstCnt += NumOfSelectInsts + /* and */ 1;
2619  }
2620  }
2621 
2622  return Res.getNode();
2623  }
2624 
2625  SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) {
2626  // Fill in BitGroups.
2627  collectBitGroups(LateMask);
2628  if (BitGroups.empty())
2629  return nullptr;
2630 
2631  // For 64-bit values, figure out when we can use 32-bit instructions.
2632  if (Bits.size() == 64)
2633  assignRepl32BitGroups();
2634 
2635  // Fill in ValueRotsVec.
2636  collectValueRotInfo();
2637 
2638  if (Bits.size() == 32) {
2639  return Select32(N, LateMask, InstCnt);
2640  } else {
2641  assert(Bits.size() == 64 && "Not 64 bits here?");
2642  return Select64(N, LateMask, InstCnt);
2643  }
2644 
2645  return nullptr;
2646  }
2647 
2648  void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) {
2649  erase_if(BitGroups, F);
2650  }
2651 
2653 
2654  bool NeedMask = false;
2656 
2657  SmallVector<BitGroup, 16> BitGroups;
2658 
2659  DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots;
2660  SmallVector<ValueRotInfo, 16> ValueRotsVec;
2661 
2662  SelectionDAG *CurDAG = nullptr;
2663 
2664 public:
2665  BitPermutationSelector(SelectionDAG *DAG)
2666  : CurDAG(DAG) {}
2667 
2668  // Here we try to match complex bit permutations into a set of
2669  // rotate-and-shift/shift/and/or instructions, using a set of heuristics
2670  // known to produce optimal code for common cases (like i32 byte swapping).
2671  SDNode *Select(SDNode *N) {
2672  Memoizer.clear();
2673  auto Result =
2674  getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits());
2675  if (!Result.first)
2676  return nullptr;
2677  Bits = std::move(*Result.second);
2678 
2679  LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction"
2680  " selection for: ");
2681  LLVM_DEBUG(N->dump(CurDAG));
2682 
2683  // Fill it RLAmt and set NeedMask.
2684  computeRotationAmounts();
2685 
2686  if (!NeedMask)
2687  return Select(N, false);
2688 
2689  // We currently have two techniques for handling results with zeros: early
2690  // masking (the default) and late masking. Late masking is sometimes more
2691  // efficient, but because the structure of the bit groups is different, it
2692  // is hard to tell without generating both and comparing the results. With
2693  // late masking, we ignore zeros in the resulting value when inserting each
2694  // set of bit groups, and then mask in the zeros at the end. With early
2695  // masking, we only insert the non-zero parts of the result at every step.
2696 
2697  unsigned InstCnt = 0, InstCntLateMask = 0;
2698  LLVM_DEBUG(dbgs() << "\tEarly masking:\n");
2699  SDNode *RN = Select(N, false, &InstCnt);
2700  LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n");
2701 
2702  LLVM_DEBUG(dbgs() << "\tLate masking:\n");
2703  SDNode *RNLM = Select(N, true, &InstCntLateMask);
2704  LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask
2705  << " instructions\n");
2706 
2707  if (InstCnt <= InstCntLateMask) {
2708  LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n");
2709  return RN;
2710  }
2711 
2712  LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n");
2713  return RNLM;
2714  }
2715 };
2716 
2717 class IntegerCompareEliminator {
2718  SelectionDAG *CurDAG;
2719  PPCDAGToDAGISel *S;
2720  // Conversion type for interpreting results of a 32-bit instruction as
2721  // a 64-bit value or vice versa.
2722  enum ExtOrTruncConversion { Ext, Trunc };
2723 
2724  // Modifiers to guide how an ISD::SETCC node's result is to be computed
2725  // in a GPR.
2726  // ZExtOrig - use the original condition code, zero-extend value
2727  // ZExtInvert - invert the condition code, zero-extend value
2728  // SExtOrig - use the original condition code, sign-extend value
2729  // SExtInvert - invert the condition code, sign-extend value
2730  enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert };
2731 
2732  // Comparisons against zero to emit GPR code sequences for. Each of these
2733  // sequences may need to be emitted for two or more equivalent patterns.
2734  // For example (a >= 0) == (a > -1). The direction of the comparison (</>)
2735  // matters as well as the extension type: sext (-1/0), zext (1/0).
2736  // GEZExt - (zext (LHS >= 0))
2737  // GESExt - (sext (LHS >= 0))
2738  // LEZExt - (zext (LHS <= 0))
2739  // LESExt - (sext (LHS <= 0))
2740  enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt };
2741 
2742  SDNode *tryEXTEND(SDNode *N);
2743  SDNode *tryLogicOpOfCompares(SDNode *N);
2744  SDValue computeLogicOpInGPR(SDValue LogicOp);
2745  SDValue signExtendInputIfNeeded(SDValue Input);
2746  SDValue zeroExtendInputIfNeeded(SDValue Input);
2747  SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv);
2748  SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2749  ZeroCompare CmpTy);
2750  SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2751  int64_t RHSValue, SDLoc dl);
2752  SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2753  int64_t RHSValue, SDLoc dl);
2754  SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2755  int64_t RHSValue, SDLoc dl);
2756  SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2757  int64_t RHSValue, SDLoc dl);
2758  SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts);
2759 
2760 public:
2761  IntegerCompareEliminator(SelectionDAG *DAG,
2762  PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) {
2763  assert(CurDAG->getTargetLoweringInfo()
2764  .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 &&
2765  "Only expecting to use this on 64 bit targets.");
2766  }
2767  SDNode *Select(SDNode *N) {
2768  if (CmpInGPR == ICGPR_None)
2769  return nullptr;
2770  switch (N->getOpcode()) {
2771  default: break;
2772  case ISD::ZERO_EXTEND:
2773  if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 ||
2775  return nullptr;
2776  [[fallthrough]];
2777  case ISD::SIGN_EXTEND:
2778  if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 ||
2780  return nullptr;
2781  return tryEXTEND(N);
2782  case ISD::AND:
2783  case ISD::OR:
2784  case ISD::XOR:
2785  return tryLogicOpOfCompares(N);
2786  }
2787  return nullptr;
2788  }
2789 };
2790 
2791 static bool isLogicOp(unsigned Opc) {
2792  return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR;
2793 }
2794 // The obvious case for wanting to keep the value in a GPR. Namely, the
2795 // result of the comparison is actually needed in a GPR.
2796 SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) {
2797  assert((N->getOpcode() == ISD::ZERO_EXTEND ||
2798  N->getOpcode() == ISD::SIGN_EXTEND) &&
2799  "Expecting a zero/sign extend node!");
2800  SDValue WideRes;
2801  // If we are zero-extending the result of a logical operation on i1
2802  // values, we can keep the values in GPRs.
2803  if (isLogicOp(N->getOperand(0).getOpcode()) &&
2804  N->getOperand(0).getValueType() == MVT::i1 &&
2805  N->getOpcode() == ISD::ZERO_EXTEND)
2806  WideRes = computeLogicOpInGPR(N->getOperand(0));
2807  else if (N->getOperand(0).getOpcode() != ISD::SETCC)
2808  return nullptr;
2809  else
2810  WideRes =
2811  getSETCCInGPR(N->getOperand(0),
2812  N->getOpcode() == ISD::SIGN_EXTEND ?
2813  SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig);
2814 
2815  if (!WideRes)
2816  return nullptr;
2817 
2818  SDLoc dl(N);
2819  bool Input32Bit = WideRes.getValueType() == MVT::i32;
2820  bool Output32Bit = N->getValueType(0) == MVT::i32;
2821 
2822  NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0;
2823  NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1;
2824 
2825  SDValue ConvOp = WideRes;
2826  if (Input32Bit != Output32Bit)
2827  ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext :
2828  ExtOrTruncConversion::Trunc);
2829  return ConvOp.getNode();
2830 }
2831 
2832 // Attempt to perform logical operations on the results of comparisons while
2833 // keeping the values in GPRs. Without doing so, these would end up being
2834 // lowered to CR-logical operations which suffer from significant latency and
2835 // low ILP.
2836 SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) {
2837  if (N->getValueType(0) != MVT::i1)
2838  return nullptr;
2839  assert(isLogicOp(N->getOpcode()) &&
2840  "Expected a logic operation on setcc results.");
2841  SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0));
2842  if (!LoweredLogical)
2843  return nullptr;
2844 
2845  SDLoc dl(N);
2846  bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8;
2847  unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt;
2848  SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
2849  SDValue LHS = LoweredLogical.getOperand(0);
2850  SDValue RHS = LoweredLogical.getOperand(1);
2851  SDValue WideOp;
2852  SDValue OpToConvToRecForm;
2853 
2854  // Look through any 32-bit to 64-bit implicit extend nodes to find the
2855  // opcode that is input to the XORI.
2856  if (IsBitwiseNegate &&
2857  LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG)
2858  OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1);
2859  else if (IsBitwiseNegate)
2860  // If the input to the XORI isn't an extension, that's what we're after.
2861  OpToConvToRecForm = LoweredLogical.getOperand(0);
2862  else
2863  // If this is not an XORI, it is a reg-reg logical op and we can convert
2864  // it to record-form.
2865  OpToConvToRecForm = LoweredLogical;
2866 
2867  // Get the record-form version of the node we're looking to use to get the
2868  // CR result from.
2869  uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode();
2870  int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc);
2871 
2872  // Convert the right node to record-form. This is either the logical we're
2873  // looking at or it is the input node to the negation (if we're looking at
2874  // a bitwise negation).
2875  if (NewOpc != -1 && IsBitwiseNegate) {
2876  // The input to the XORI has a record-form. Use it.
2877  assert(LoweredLogical.getConstantOperandVal(1) == 1 &&
2878  "Expected a PPC::XORI8 only for bitwise negation.");
2879  // Emit the record-form instruction.
2880  std::vector<SDValue> Ops;
2881  for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++)
2882  Ops.push_back(OpToConvToRecForm.getOperand(i));
2883 
2884  WideOp =
2885  SDValue(CurDAG->getMachineNode(NewOpc, dl,
2886  OpToConvToRecForm.getValueType(),
2887  MVT::Glue, Ops), 0);
2888  } else {
2889  assert((NewOpc != -1 || !IsBitwiseNegate) &&
2890  "No record form available for AND8/OR8/XOR8?");
2891  WideOp =
2892  SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc,
2893  dl, MVT::i64, MVT::Glue, LHS, RHS),
2894  0);
2895  }
2896 
2897  // Select this node to a single bit from CR0 set by the record-form node
2898  // just created. For bitwise negation, use the EQ bit which is the equivalent
2899  // of negating the result (i.e. it is a bit set when the result of the
2900  // operation is zero).
2901  SDValue SRIdxVal =
2902  CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32);
2903  SDValue CRBit =
2904  SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
2905  MVT::i1, CR0Reg, SRIdxVal,
2906  WideOp.getValue(1)), 0);
2907  return CRBit.getNode();
2908 }
2909 
2910 // Lower a logical operation on i1 values into a GPR sequence if possible.
2911 // The result can be kept in a GPR if requested.
2912 // Three types of inputs can be handled:
2913 // - SETCC
2914 // - TRUNCATE
2915 // - Logical operation (AND/OR/XOR)
2916 // There is also a special case that is handled (namely a complement operation
2917 // achieved with xor %a, -1).
2918 SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) {
2919  assert(isLogicOp(LogicOp.getOpcode()) &&
2920  "Can only handle logic operations here.");
2921  assert(LogicOp.getValueType() == MVT::i1 &&
2922  "Can only handle logic operations on i1 values here.");
2923  SDLoc dl(LogicOp);
2924  SDValue LHS, RHS;
2925 
2926  // Special case: xor %a, -1
2927  bool IsBitwiseNegation = isBitwiseNot(LogicOp);
2928 
2929  // Produces a GPR sequence for each operand of the binary logic operation.
2930  // For SETCC, it produces the respective comparison, for TRUNCATE it truncates
2931  // the value in a GPR and for logic operations, it will recursively produce
2932  // a GPR sequence for the operation.
2933  auto getLogicOperand = [&] (SDValue Operand) -> SDValue {
2934  unsigned OperandOpcode = Operand.getOpcode();
2935  if (OperandOpcode == ISD::SETCC)
2936  return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig);
2937  else if (OperandOpcode == ISD::TRUNCATE) {
2938  SDValue InputOp = Operand.getOperand(0);
2939  EVT InVT = InputOp.getValueType();
2940  return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 :
2941  PPC::RLDICL, dl, InVT, InputOp,
2942  S->getI64Imm(0, dl),
2943  S->getI64Imm(63, dl)), 0);
2944  } else if (isLogicOp(OperandOpcode))
2945  return computeLogicOpInGPR(Operand);
2946  return SDValue();
2947  };
2948  LHS = getLogicOperand(LogicOp.getOperand(0));
2949  RHS = getLogicOperand(LogicOp.getOperand(1));
2950 
2951  // If a GPR sequence can't be produced for the LHS we can't proceed.
2952  // Not producing a GPR sequence for the RHS is only a problem if this isn't
2953  // a bitwise negation operation.
2954  if (!LHS || (!RHS && !IsBitwiseNegation))
2955  return SDValue();
2956 
2957  NumLogicOpsOnComparison++;
2958 
2959  // We will use the inputs as 64-bit values.
2960  if (LHS.getValueType() == MVT::i32)
2961  LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext);
2962  if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32)
2963  RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext);
2964 
2965  unsigned NewOpc;
2966  switch (LogicOp.getOpcode()) {
2967  default: llvm_unreachable("Unknown logic operation.");
2968  case ISD::AND: NewOpc = PPC::AND8; break;
2969  case ISD::OR: NewOpc = PPC::OR8; break;
2970  case ISD::XOR: NewOpc = PPC::XOR8; break;
2971  }
2972 
2973  if (IsBitwiseNegation) {
2974  RHS = S->getI64Imm(1, dl);
2975  NewOpc = PPC::XORI8;
2976  }
2977 
2978  return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0);
2979 
2980 }
2981 
2982 /// If the value isn't guaranteed to be sign-extended to 64-bits, extend it.
2983 /// Otherwise just reinterpret it as a 64-bit value.
2984 /// Useful when emitting comparison code for 32-bit values without using
2985 /// the compare instruction (which only considers the lower 32-bits).
2986 SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) {
2987  assert(Input.getValueType() == MVT::i32 &&
2988  "Can only sign-extend 32-bit values here.");
2989  unsigned Opc = Input.getOpcode();
2990 
2991  // The value was sign extended and then truncated to 32-bits. No need to
2992  // sign extend it again.
2993  if (Opc == ISD::TRUNCATE &&
2994  (Input.getOperand(0).getOpcode() == ISD::AssertSext ||
2995  Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND))
2996  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2997 
2998  LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
2999  // The input is a sign-extending load. All ppc sign-extending loads
3000  // sign-extend to the full 64-bits.
3001  if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD)
3002  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3003 
3004  ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
3005  // We don't sign-extend constants.
3006  if (InputConst)
3007  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3008 
3009  SDLoc dl(Input);
3010  SignExtensionsAdded++;
3011  return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl,
3012  MVT::i64, Input), 0);
3013 }
3014 
3015 /// If the value isn't guaranteed to be zero-extended to 64-bits, extend it.
3016 /// Otherwise just reinterpret it as a 64-bit value.
3017 /// Useful when emitting comparison code for 32-bit values without using
3018 /// the compare instruction (which only considers the lower 32-bits).
3019 SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) {
3020  assert(Input.getValueType() == MVT::i32 &&
3021  "Can only zero-extend 32-bit values here.");
3022  unsigned Opc = Input.getOpcode();
3023 
3024  // The only condition under which we can omit the actual extend instruction:
3025  // - The value is a positive constant
3026  // - The value comes from a load that isn't a sign-extending load
3027  // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext.
3028  bool IsTruncateOfZExt = Opc == ISD::TRUNCATE &&
3029  (Input.getOperand(0).getOpcode() == ISD::AssertZext ||
3030  Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND);
3031  if (IsTruncateOfZExt)
3032  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3033 
3034  ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
3035  if (InputConst && InputConst->getSExtValue() >= 0)
3036  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3037 
3038  LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
3039  // The input is a load that doesn't sign-extend (it will be zero-extended).
3040  if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD)
3041  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3042 
3043  // None of the above, need to zero-extend.
3044  SDLoc dl(Input);
3045  ZeroExtensionsAdded++;
3046  return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input,
3047  S->getI64Imm(0, dl),
3048  S->getI64Imm(32, dl)), 0);
3049 }
3050 
3051 // Handle a 32-bit value in a 64-bit register and vice-versa. These are of
3052 // course not actual zero/sign extensions that will generate machine code,
3053 // they're just a way to reinterpret a 32 bit value in a register as a
3054 // 64 bit value and vice-versa.
3055 SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes,
3056  ExtOrTruncConversion Conv) {
3057  SDLoc dl(NatWidthRes);
3058 
3059  // For reinterpreting 32-bit values as 64 bit values, we generate
3060  // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1>
3061  if (Conv == ExtOrTruncConversion::Ext) {
3062  SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0);
3063  SDValue SubRegIdx =
3064  CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
3065  return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64,
3066  ImDef, NatWidthRes, SubRegIdx), 0);
3067  }
3068 
3069  assert(Conv == ExtOrTruncConversion::Trunc &&
3070  "Unknown convertion between 32 and 64 bit values.");
3071  // For reinterpreting 64-bit values as 32-bit values, we just need to
3072  // EXTRACT_SUBREG (i.e. extract the low word).
3073  SDValue SubRegIdx =
3074  CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
3075  return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32,
3076  NatWidthRes, SubRegIdx), 0);
3077 }
3078 
3079 // Produce a GPR sequence for compound comparisons (<=, >=) against zero.
3080 // Handle both zero-extensions and sign-extensions.
3081 SDValue
3082 IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
3083  ZeroCompare CmpTy) {
3084  EVT InVT = LHS.getValueType();
3085  bool Is32Bit = InVT == MVT::i32;
3086  SDValue ToExtend;
3087 
3088  // Produce the value that needs to be either zero or sign extended.
3089  switch (CmpTy) {
3090  case ZeroCompare::GEZExt:
3091  case ZeroCompare::GESExt:
3092  ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8,
3093  dl, InVT, LHS, LHS), 0);
3094  break;
3095  case ZeroCompare::LEZExt:
3096  case ZeroCompare::LESExt: {
3097  if (Is32Bit) {
3098  // Upper 32 bits cannot be undefined for this sequence.
3099  LHS = signExtendInputIfNeeded(LHS);
3100  SDValue Neg =
3101  SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3102  ToExtend =
3103  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3104  Neg, S->getI64Imm(1, dl),
3105  S->getI64Imm(63, dl)), 0);
3106  } else {
3107  SDValue Addi =
3108  SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3109  S->getI64Imm(~0ULL, dl)), 0);
3110  ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
3111  Addi, LHS), 0);
3112  }
3113  break;
3114  }
3115  }
3116 
3117  // For 64-bit sequences, the extensions are the same for the GE/LE cases.
3118  if (!Is32Bit &&
3119  (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt))
3120  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3121  ToExtend, S->getI64Imm(1, dl),
3122  S->getI64Imm(63, dl)), 0);
3123  if (!Is32Bit &&
3124  (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt))
3125  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend,
3126  S->getI64Imm(63, dl)), 0);
3127 
3128  assert(Is32Bit && "Should have handled the 32-bit sequences above.");
3129  // For 32-bit sequences, the extensions differ between GE/LE cases.
3130  switch (CmpTy) {
3131  case ZeroCompare::GEZExt: {
3132  SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3133  S->getI32Imm(31, dl) };
3134  return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3135  ShiftOps), 0);
3136  }
3137  case ZeroCompare::GESExt:
3138  return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend,
3139  S->getI32Imm(31, dl)), 0);
3140  case ZeroCompare::LEZExt:
3141  return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend,
3142  S->getI32Imm(1, dl)), 0);
3143  case ZeroCompare::LESExt:
3144  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend,
3145  S->getI32Imm(-1, dl)), 0);
3146  }
3147 
3148  // The above case covers all the enumerators so it can't have a default clause
3149  // to avoid compiler warnings.
3150  llvm_unreachable("Unknown zero-comparison type.");
3151 }
3152 
3153 /// Produces a zero-extended result of comparing two 32-bit values according to
3154 /// the passed condition code.
3155 SDValue
3156 IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS,
3157  ISD::CondCode CC,
3158  int64_t RHSValue, SDLoc dl) {
3159  if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
3161  return SDValue();
3162  bool IsRHSZero = RHSValue == 0;
3163  bool IsRHSOne = RHSValue == 1;
3164  bool IsRHSNegOne = RHSValue == -1LL;
3165  switch (CC) {
3166  default: return SDValue();
3167  case ISD::SETEQ: {
3168  // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5)
3169  // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5)
3170  SDValue Xor = IsRHSZero ? LHS :
3171  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3172  SDValue Clz =
3173  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3174  SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3175  S->getI32Imm(31, dl) };
3176  return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3177  ShiftOps), 0);
3178  }
3179  case ISD::SETNE: {
3180  // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1)
3181  // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1)
3182  SDValue Xor = IsRHSZero ? LHS :
3183  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3184  SDValue Clz =
3185  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3186  SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3187  S->getI32Imm(31, dl) };
3188  SDValue Shift =
3189  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3190  return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3191  S->getI32Imm(1, dl)), 0);
3192  }
3193  case ISD::SETGE: {
3194  // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1)
3195  // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31)
3196  if(IsRHSZero)
3197  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3198 
3199  // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3200  // by swapping inputs and falling through.
3201  std::swap(LHS, RHS);
3202  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3203  IsRHSZero = RHSConst && RHSConst->isZero();
3204  [[fallthrough]];
3205  }
3206  case ISD::SETLE: {
3207  if (CmpInGPR == ICGPR_NonExtIn)
3208  return SDValue();
3209  // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1)
3210  // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1)
3211  if(IsRHSZero) {
3212  if (CmpInGPR == ICGPR_NonExtIn)
3213  return SDValue();
3214  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3215  }
3216 
3217  // The upper 32-bits of the register can't be undefined for this sequence.
3218  LHS = signExtendInputIfNeeded(LHS);
3219  RHS = signExtendInputIfNeeded(RHS);
3220  SDValue Sub =
3221  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3222  SDValue Shift =
3223  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub,
3224  S->getI64Imm(1, dl), S->getI64Imm(63, dl)),
3225  0);
3226  return
3227  SDValue(CurDAG->getMachineNode(PPC::XORI8, dl,
3228  MVT::i64, Shift, S->getI32Imm(1, dl)), 0);
3229  }
3230  case ISD::SETGT: {
3231  // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63)
3232  // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31)
3233  // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63)
3234  // Handle SETLT -1 (which is equivalent to SETGE 0).
3235  if (IsRHSNegOne)
3236  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3237 
3238  if (IsRHSZero) {
3239  if (CmpInGPR == ICGPR_NonExtIn)
3240  return SDValue();
3241  // The upper 32-bits of the register can't be undefined for this sequence.
3242  LHS = signExtendInputIfNeeded(LHS);
3243  RHS = signExtendInputIfNeeded(RHS);
3244  SDValue Neg =
3245  SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3246  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3247  Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0);
3248  }
3249  // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3250  // (%b < %a) by swapping inputs and falling through.
3251  std::swap(LHS, RHS);
3252  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3253  IsRHSZero = RHSConst && RHSConst->isZero();
3254  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3255  [[fallthrough]];
3256  }
3257  case ISD::SETLT: {
3258  // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63)
3259  // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1)
3260  // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31)
3261  // Handle SETLT 1 (which is equivalent to SETLE 0).
3262  if (IsRHSOne) {
3263  if (CmpInGPR == ICGPR_NonExtIn)
3264  return SDValue();
3265  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3266  }
3267 
3268  if (IsRHSZero) {
3269  SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3270  S->getI32Imm(31, dl) };
3271  return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3272  ShiftOps), 0);
3273  }
3274 
3275  if (CmpInGPR == ICGPR_NonExtIn)
3276  return SDValue();
3277  // The upper 32-bits of the register can't be undefined for this sequence.
3278  LHS = signExtendInputIfNeeded(LHS);
3279  RHS = signExtendInputIfNeeded(RHS);
3280  SDValue SUBFNode =
3281  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3282  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3283  SUBFNode, S->getI64Imm(1, dl),
3284  S->getI64Imm(63, dl)), 0);
3285  }
3286  case ISD::SETUGE:
3287  // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1)
3288  // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1)
3289  std::swap(LHS, RHS);
3290  [[fallthrough]];
3291  case ISD::SETULE: {
3292  if (CmpInGPR == ICGPR_NonExtIn)
3293  return SDValue();
3294  // The upper 32-bits of the register can't be undefined for this sequence.
3295  LHS = zeroExtendInputIfNeeded(LHS);
3296  RHS = zeroExtendInputIfNeeded(RHS);
3297  SDValue Subtract =
3298  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3299  SDValue SrdiNode =
3300  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3301  Subtract, S->getI64Imm(1, dl),
3302  S->getI64Imm(63, dl)), 0);
3303  return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode,
3304  S->getI32Imm(1, dl)), 0);
3305  }
3306  case ISD::SETUGT:
3307  // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63)
3308  // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63)
3309  std::swap(LHS, RHS);
3310  [[fallthrough]];
3311  case ISD::SETULT: {
3312  if (CmpInGPR == ICGPR_NonExtIn)
3313  return SDValue();
3314  // The upper 32-bits of the register can't be undefined for this sequence.
3315  LHS = zeroExtendInputIfNeeded(LHS);
3316  RHS = zeroExtendInputIfNeeded(RHS);
3317  SDValue Subtract =
3318  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3319  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3320  Subtract, S->getI64Imm(1, dl),
3321  S->getI64Imm(63, dl)), 0);
3322  }
3323  }
3324 }
3325 
3326 /// Produces a sign-extended result of comparing two 32-bit values according to
3327 /// the passed condition code.
3328 SDValue
3329 IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS,
3330  ISD::CondCode CC,
3331  int64_t RHSValue, SDLoc dl) {
3332  if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
3334  return SDValue();
3335  bool IsRHSZero = RHSValue == 0;
3336  bool IsRHSOne = RHSValue == 1;
3337  bool IsRHSNegOne = RHSValue == -1LL;
3338 
3339  switch (CC) {
3340  default: return SDValue();
3341  case ISD::SETEQ: {
3342  // (sext (setcc %a, %b, seteq)) ->
3343  // (ashr (shl (ctlz (xor %a, %b)), 58), 63)
3344  // (sext (setcc %a, 0, seteq)) ->
3345  // (ashr (shl (ctlz %a), 58), 63)
3346  SDValue CountInput = IsRHSZero ? LHS :
3347  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3348  SDValue Cntlzw =
3349  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0);
3350  SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl),
3351  S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3352  SDValue Slwi =
3353  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0);
3354  return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0);
3355  }
3356  case ISD::SETNE: {
3357  // Bitwise xor the operands, count leading zeros, shift right by 5 bits and
3358  // flip the bit, finally take 2's complement.
3359  // (sext (setcc %a, %b, setne)) ->
3360  // (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1))
3361  // Same as above, but the first xor is not needed.
3362  // (sext (setcc %a, 0, setne)) ->
3363  // (neg (xor (lshr (ctlz %a), 5), 1))
3364  SDValue Xor = IsRHSZero ? LHS :
3365  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3366  SDValue Clz =
3367  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3368  SDValue ShiftOps[] =
3369  { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3370  SDValue Shift =
3371  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3372  SDValue Xori =
3373  SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3374  S->getI32Imm(1, dl)), 0);
3375  return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0);
3376  }
3377  case ISD::SETGE: {
3378  // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1)
3379  // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31)
3380  if (IsRHSZero)
3381  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3382 
3383  // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3384  // by swapping inputs and falling through.
3385  std::swap(LHS, RHS);
3386  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3387  IsRHSZero = RHSConst && RHSConst->isZero();
3388  [[fallthrough]];
3389  }
3390  case ISD::SETLE: {
3391  if (CmpInGPR == ICGPR_NonExtIn)
3392  return SDValue();
3393  // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1)
3394  // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1)
3395  if (IsRHSZero)
3396  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3397 
3398  // The upper 32-bits of the register can't be undefined for this sequence.
3399  LHS = signExtendInputIfNeeded(LHS);
3400  RHS = signExtendInputIfNeeded(RHS);
3401  SDValue SUBFNode =
3402  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue,
3403  LHS, RHS), 0);
3404  SDValue Srdi =
3405  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3406  SUBFNode, S->getI64Imm(1, dl),
3407  S->getI64Imm(63, dl)), 0);
3408  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi,
3409  S->getI32Imm(-1, dl)), 0);
3410  }
3411  case ISD::SETGT: {
3412  // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63)
3413  // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31)
3414  // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63)
3415  if (IsRHSNegOne)
3416  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3417  if (IsRHSZero) {
3418  if (CmpInGPR == ICGPR_NonExtIn)
3419  return SDValue();
3420  // The upper 32-bits of the register can't be undefined for this sequence.
3421  LHS = signExtendInputIfNeeded(LHS);
3422  RHS = signExtendInputIfNeeded(RHS);
3423  SDValue Neg =
3424  SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3425  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg,
3426  S->getI64Imm(63, dl)), 0);
3427  }
3428  // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3429  // (%b < %a) by swapping inputs and falling through.
3430  std::swap(LHS, RHS);
3431  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3432  IsRHSZero = RHSConst && RHSConst->isZero();
3433  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3434  [[fallthrough]];
3435  }
3436  case ISD::SETLT: {
3437  // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63)
3438  // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1)
3439  // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31)
3440  if (IsRHSOne) {
3441  if (CmpInGPR == ICGPR_NonExtIn)
3442  return SDValue();
3443  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3444  }
3445  if (IsRHSZero)
3446  return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS,
3447  S->getI32Imm(31, dl)), 0);
3448 
3449  if (CmpInGPR == ICGPR_NonExtIn)
3450  return SDValue();
3451  // The upper 32-bits of the register can't be undefined for this sequence.
3452  LHS = signExtendInputIfNeeded(LHS);
3453  RHS = signExtendInputIfNeeded(RHS);
3454  SDValue SUBFNode =
3455  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3456  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3457  SUBFNode, S->getI64Imm(63, dl)), 0);
3458  }
3459  case ISD::SETUGE:
3460  // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1)
3461  // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1)
3462  std::swap(LHS, RHS);
3463  [[fallthrough]];
3464  case ISD::SETULE: {
3465  if (CmpInGPR == ICGPR_NonExtIn)
3466  return SDValue();
3467  // The upper 32-bits of the register can't be undefined for this sequence.
3468  LHS = zeroExtendInputIfNeeded(LHS);
3469  RHS = zeroExtendInputIfNeeded(RHS);
3470  SDValue Subtract =
3471  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3472  SDValue Shift =
3473  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract,
3474  S->getI32Imm(1, dl), S->getI32Imm(63,dl)),
3475  0);
3476  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift,
3477  S->getI32Imm(-1, dl)), 0);
3478  }
3479  case ISD::SETUGT:
3480  // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63)
3481  // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63)
3482  std::swap(LHS, RHS);
3483  [[fallthrough]];
3484  case ISD::SETULT: {
3485  if (CmpInGPR == ICGPR_NonExtIn)
3486  return SDValue();
3487  // The upper 32-bits of the register can't be undefined for this sequence.
3488  LHS = zeroExtendInputIfNeeded(LHS);
3489  RHS = zeroExtendInputIfNeeded(RHS);
3490  SDValue Subtract =
3491  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3492  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3493  Subtract, S->getI64Imm(63, dl)), 0);
3494  }
3495  }
3496 }
3497 
3498 /// Produces a zero-extended result of comparing two 64-bit values according to
3499 /// the passed condition code.
3500 SDValue
3501 IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS,
3502  ISD::CondCode CC,
3503  int64_t RHSValue, SDLoc dl) {
3504  if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3506  return SDValue();
3507  bool IsRHSZero = RHSValue == 0;
3508  bool IsRHSOne = RHSValue == 1;
3509  bool IsRHSNegOne = RHSValue == -1LL;
3510  switch (CC) {
3511  default: return SDValue();
3512  case ISD::SETEQ: {
3513  // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6)
3514  // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6)
3515  SDValue Xor = IsRHSZero ? LHS :
3516  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3517  SDValue Clz =
3518  SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0);
3519  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz,
3520  S->getI64Imm(58, dl),
3521  S->getI64Imm(63, dl)), 0);
3522  }
3523  case ISD::SETNE: {
3524  // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3525  // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA)
3526  // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3527  // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3528  SDValue Xor = IsRHSZero ? LHS :
3529  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3530  SDValue AC =
3531  SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3532  Xor, S->getI32Imm(~0U, dl)), 0);
3533  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC,
3534  Xor, AC.getValue(1)), 0);
3535  }
3536  case ISD::SETGE: {
3537  // {subc.reg, subc.CA} = (subcarry %a, %b)
3538  // (zext (setcc %a, %b, setge)) ->
3539  // (adde (lshr %b, 63), (ashr %a, 63), subc.CA)
3540  // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63)
3541  if (IsRHSZero)
3542  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3543  std::swap(LHS, RHS);
3544  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3545  IsRHSZero = RHSConst && RHSConst->isZero();
3546  [[fallthrough]];
3547  }
3548  case ISD::SETLE: {
3549  // {subc.reg, subc.CA} = (subcarry %b, %a)
3550  // (zext (setcc %a, %b, setge)) ->
3551  // (adde (lshr %a, 63), (ashr %b, 63), subc.CA)
3552  // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63)
3553  if (IsRHSZero)
3554  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3555  SDValue ShiftL =
3556  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3557  S->getI64Imm(1, dl),
3558  S->getI64Imm(63, dl)), 0);
3559  SDValue ShiftR =
3560  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3561  S->getI64Imm(63, dl)), 0);
3562  SDValue SubtractCarry =
3563  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3564  LHS, RHS), 1);
3565  return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3566  ShiftR, ShiftL, SubtractCarry), 0);
3567  }
3568  case ISD::SETGT: {
3569  // {subc.reg, subc.CA} = (subcarry %b, %a)
3570  // (zext (setcc %a, %b, setgt)) ->
3571  // (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3572  // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63)
3573  if (IsRHSNegOne)
3574  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3575  if (IsRHSZero) {
3576  SDValue Addi =
3577  SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3578  S->getI64Imm(~0ULL, dl)), 0);
3579  SDValue Nor =
3580  SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0);
3581  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor,
3582  S->getI64Imm(1, dl),
3583  S->getI64Imm(63, dl)), 0);
3584  }
3585  std::swap(LHS, RHS);
3586  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3587  IsRHSZero = RHSConst && RHSConst->isZero();
3588  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3589  [[fallthrough]];
3590  }
3591  case ISD::SETLT: {
3592  // {subc.reg, subc.CA} = (subcarry %a, %b)
3593  // (zext (setcc %a, %b, setlt)) ->
3594  // (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3595  // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63)
3596  if (IsRHSOne)
3597  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3598  if (IsRHSZero)
3599  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3600  S->getI64Imm(1, dl),
3601  S->getI64Imm(63, dl)), 0);
3602  SDValue SRADINode =
3603  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3604  LHS, S->getI64Imm(63, dl)), 0);
3605  SDValue SRDINode =
3606  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3607  RHS, S->getI64Imm(1, dl),
3608  S->getI64Imm(63, dl)), 0);
3609  SDValue SUBFC8Carry =
3610  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3611  RHS, LHS), 1);
3612  SDValue ADDE8Node =
3613  SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3614  SRDINode, SRADINode, SUBFC8Carry), 0);
3615  return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3616  ADDE8Node, S->getI64Imm(1, dl)), 0);
3617  }
3618  case ISD::SETUGE:
3619  // {subc.reg, subc.CA} = (subcarry %a, %b)
3620  // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1)
3621  std::swap(LHS, RHS);
3622  [[fallthrough]];
3623  case ISD::SETULE: {
3624  // {subc.reg, subc.CA} = (subcarry %b, %a)
3625  // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1)
3626  SDValue SUBFC8Carry =
3627  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3628  LHS, RHS), 1);
3629  SDValue SUBFE8Node =
3630  SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue,
3631  LHS, LHS, SUBFC8Carry), 0);
3632  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64,
3633  SUBFE8Node, S->getI64Imm(1, dl)), 0);
3634  }
3635  case ISD::SETUGT:
3636  // {subc.reg, subc.CA} = (subcarry %b, %a)
3637  // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA)
3638  std::swap(LHS, RHS);
3639  [[fallthrough]];
3640  case ISD::SETULT: {
3641  // {subc.reg, subc.CA} = (subcarry %a, %b)
3642  // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA)
3643  SDValue SubtractCarry =
3644  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3645  RHS, LHS), 1);
3646  SDValue ExtSub =
3647  SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3648  LHS, LHS, SubtractCarry), 0);
3649  return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3650  ExtSub), 0);
3651  }
3652  }
3653 }
3654 
3655 /// Produces a sign-extended result of comparing two 64-bit values according to
3656 /// the passed condition code.
3657 SDValue
3658 IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS,
3659  ISD::CondCode CC,
3660  int64_t RHSValue, SDLoc dl) {
3661  if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3663  return SDValue();
3664  bool IsRHSZero = RHSValue == 0;
3665  bool IsRHSOne = RHSValue == 1;
3666  bool IsRHSNegOne = RHSValue == -1LL;
3667  switch (CC) {
3668  default: return SDValue();
3669  case ISD::SETEQ: {
3670  // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3671  // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA)
3672  // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3673  // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3674  SDValue AddInput = IsRHSZero ? LHS :
3675  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3676  SDValue Addic =
3677  SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3678  AddInput, S->getI32Imm(~0U, dl)), 0);
3679  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic,
3680  Addic, Addic.getValue(1)), 0);
3681  }
3682  case ISD::SETNE: {
3683  // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b))
3684  // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA)
3685  // {subfcz.reg, subfcz.CA} = (subcarry 0, %a)
3686  // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA)
3687  SDValue Xor = IsRHSZero ? LHS :
3688  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3689  SDValue SC =
3690  SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue,
3691  Xor, S->getI32Imm(0, dl)), 0);
3692  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC,
3693  SC, SC.getValue(1)), 0);
3694  }
3695  case ISD::SETGE: {
3696  // {subc.reg, subc.CA} = (subcarry %a, %b)
3697  // (zext (setcc %a, %b, setge)) ->
3698  // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA))
3699  // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63))
3700  if (IsRHSZero)
3701  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3702  std::swap(LHS, RHS);
3703  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3704  IsRHSZero = RHSConst && RHSConst->isZero();
3705  [[fallthrough]];
3706  }
3707  case ISD::SETLE: {
3708  // {subc.reg, subc.CA} = (subcarry %b, %a)
3709  // (zext (setcc %a, %b, setge)) ->
3710  // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA))
3711  // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63)
3712  if (IsRHSZero)
3713  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3714  SDValue ShiftR =
3715  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3716  S->getI64Imm(63, dl)), 0);
3717  SDValue ShiftL =
3718  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3719  S->getI64Imm(1, dl),
3720  S->getI64Imm(63, dl)), 0);
3721  SDValue SubtractCarry =
3722  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3723  LHS, RHS), 1);
3724  SDValue Adde =
3725  SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3726  ShiftR, ShiftL, SubtractCarry), 0);
3727  return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0);
3728  }
3729  case ISD::SETGT: {
3730  // {subc.reg, subc.CA} = (subcarry %b, %a)
3731  // (zext (setcc %a, %b, setgt)) ->
3732  // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3733  // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63)
3734  if (IsRHSNegOne)
3735  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3736  if (IsRHSZero) {
3737  SDValue Add =
3738  SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3739  S->getI64Imm(-1, dl)), 0);
3740  SDValue Nor =
3741  SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0);
3742  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor,
3743  S->getI64Imm(63, dl)), 0);
3744  }
3745  std::swap(LHS, RHS);
3746  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3747  IsRHSZero = RHSConst && RHSConst->isZero();
3748  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3749  [[fallthrough]];
3750  }
3751  case ISD::SETLT: {
3752  // {subc.reg, subc.CA} = (subcarry %a, %b)
3753  // (zext (setcc %a, %b, setlt)) ->
3754  // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3755  // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63)
3756  if (IsRHSOne)
3757  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3758  if (IsRHSZero) {
3759  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS,
3760  S->getI64Imm(63, dl)), 0);
3761  }
3762  SDValue SRADINode =
3763  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3764  LHS, S->getI64Imm(63, dl)), 0);
3765  SDValue SRDINode =
3766  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3767  RHS, S->getI64Imm(1, dl),
3768  S->getI64Imm(63, dl)), 0);
3769  SDValue SUBFC8Carry =
3770  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3771  RHS, LHS), 1);
3772  SDValue ADDE8Node =
3773  SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64,
3774  SRDINode, SRADINode, SUBFC8Carry), 0);
3775  SDValue XORI8Node =
3776  SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3777  ADDE8Node, S->getI64Imm(1, dl)), 0);
3778  return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3779  XORI8Node), 0);
3780  }
3781  case ISD::SETUGE:
3782  // {subc.reg, subc.CA} = (subcarry %a, %b)
3783  // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA)
3784  std::swap(LHS, RHS);
3785  [[fallthrough]];
3786  case ISD::SETULE: {
3787  // {subc.reg, subc.CA} = (subcarry %b, %a)
3788  // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA)
3789  SDValue SubtractCarry =
3790  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3791  LHS, RHS), 1);
3792  SDValue ExtSub =
3793  SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS,
3794  LHS, SubtractCarry), 0);
3795  return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64,
3796  ExtSub, ExtSub), 0);
3797  }
3798  case ISD::SETUGT:
3799  // {subc.reg, subc.CA} = (subcarry %b, %a)
3800  // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA)
3801  std::swap(LHS, RHS);
3802  [[fallthrough]];
3803  case ISD::SETULT: {
3804  // {subc.reg, subc.CA} = (subcarry %a, %b)
3805  // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA)
3806  SDValue SubCarry =
3807  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3808  RHS, LHS), 1);
3809  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3810  LHS, LHS, SubCarry), 0);
3811  }
3812  }
3813 }
3814 
3815 /// Do all uses of this SDValue need the result in a GPR?
3816 /// This is meant to be used on values that have type i1 since
3817 /// it is somewhat meaningless to ask if values of other types
3818 /// should be kept in GPR's.
3819 static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) {
3820  assert(Compare.getOpcode() == ISD::SETCC &&
3821  "An ISD::SETCC node required here.");
3822 
3823  // For values that have a single use, the caller should obviously already have
3824  // checked if that use is an extending use. We check the other uses here.
3825  if (Compare.hasOneUse())
3826  return true;
3827  // We want the value in a GPR if it is being extended, used for a select, or
3828  // used in logical operations.
3829  for (auto *CompareUse : Compare.getNode()->uses())
3830  if (CompareUse->getOpcode() != ISD::SIGN_EXTEND &&
3831  CompareUse->getOpcode() != ISD::ZERO_EXTEND &&
3832  CompareUse->getOpcode() != ISD::SELECT &&
3833  !isLogicOp(CompareUse->getOpcode())) {
3834  OmittedForNonExtendUses++;
3835  return false;
3836  }
3837  return true;
3838 }
3839 
3840 /// Returns an equivalent of a SETCC node but with the result the same width as
3841 /// the inputs. This can also be used for SELECT_CC if either the true or false
3842 /// values is a power of two while the other is zero.
3843 SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare,
3844  SetccInGPROpts ConvOpts) {
3845  assert((Compare.getOpcode() == ISD::SETCC ||
3846  Compare.getOpcode() == ISD::SELECT_CC) &&
3847  "An ISD::SETCC node required here.");
3848 
3849  // Don't convert this comparison to a GPR sequence because there are uses
3850  // of the i1 result (i.e. uses that require the result in the CR).
3851  if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG))
3852  return SDValue();
3853 
3854  SDValue LHS = Compare.getOperand(0);
3855  SDValue RHS = Compare.getOperand(1);
3856 
3857  // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC.
3858  int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2;
3859  ISD::CondCode CC =
3860  cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get();
3861  EVT InputVT = LHS.getValueType();
3862  if (InputVT != MVT::i32 && InputVT != MVT::i64)
3863  return SDValue();
3864 
3865  if (ConvOpts == SetccInGPROpts::ZExtInvert ||
3866  ConvOpts == SetccInGPROpts::SExtInvert)
3867  CC = ISD::getSetCCInverse(CC, InputVT);
3868 
3869  bool Inputs32Bit = InputVT == MVT::i32;
3870 
3871  SDLoc dl(Compare);
3872  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3873  int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX;
3874  bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig ||
3875  ConvOpts == SetccInGPROpts::SExtInvert;
3876 
3877  if (IsSext && Inputs32Bit)
3878  return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3879  else if (Inputs32Bit)
3880  return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3881  else if (IsSext)
3882  return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3883  return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3884 }
3885 
3886 } // end anonymous namespace
3887 
3888 bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) {
3889  if (N->getValueType(0) != MVT::i32 &&
3890  N->getValueType(0) != MVT::i64)
3891  return false;
3892 
3893  // This optimization will emit code that assumes 64-bit registers
3894  // so we don't want to run it in 32-bit mode. Also don't run it
3895  // on functions that are not to be optimized.
3896  if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64())
3897  return false;
3898 
3899  // For POWER10, it is more profitable to use the set boolean extension
3900  // instructions rather than the integer compare elimination codegen.
3901  // Users can override this via the command line option, `--ppc-gpr-icmps`.
3902  if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1())
3903  return false;
3904 
3905  switch (N->getOpcode()) {
3906  default: break;
3907  case ISD::ZERO_EXTEND:
3908  case ISD::SIGN_EXTEND:
3909  case ISD::AND:
3910  case ISD::OR:
3911  case ISD::XOR: {
3912  IntegerCompareEliminator ICmpElim(CurDAG, this);
3913  if (SDNode *New = ICmpElim.Select(N)) {
3914  ReplaceNode(N, New);
3915  return true;
3916  }
3917  }
3918  }
3919  return false;
3920 }
3921 
3922 bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) {
3923  if (N->getValueType(0) != MVT::i32 &&
3924  N->getValueType(0) != MVT::i64)
3925  return false;
3926 
3927  if (!UseBitPermRewriter)
3928  return false;
3929 
3930  switch (N->getOpcode()) {
3931  default: break;
3932  case ISD::ROTL:
3933  case ISD::SHL:
3934  case ISD::SRL:
3935  case ISD::AND:
3936  case ISD::OR: {
3937  BitPermutationSelector BPS(CurDAG);
3938  if (SDNode *New = BPS.Select(N)) {
3939  ReplaceNode(N, New);
3940  return true;
3941  }
3942  return false;
3943  }
3944  }
3945 
3946  return false;
3947 }
3948 
3949 /// SelectCC - Select a comparison of the specified values with the specified
3950 /// condition code, returning the CR# of the expression.
3951 SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3952  const SDLoc &dl, SDValue Chain) {
3953  // Always select the LHS.
3954  unsigned Opc;
3955 
3956  if (LHS.getValueType() == MVT::i32) {
3957  unsigned Imm;
3958  if (CC == ISD::SETEQ || CC == ISD::SETNE) {
3959  if (isInt32Immediate(RHS, Imm)) {
3960  // SETEQ/SETNE comparison with 16-bit immediate, fold it.
3961  if (isUInt<16>(Imm))
3962  return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3963  getI32Imm(Imm & 0xFFFF, dl)),
3964  0);
3965  // If this is a 16-bit signed immediate, fold it.
3966  if (isInt<16>((int)Imm))
3967  return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3968  getI32Imm(Imm & 0xFFFF, dl)),
3969  0);
3970 
3971  // For non-equality comparisons, the default code would materialize the
3972  // constant, then compare against it, like this:
3973  // lis r2, 4660
3974  // ori r2, r2, 22136
3975  // cmpw cr0, r3, r2
3976  // Since we are just comparing for equality, we can emit this instead:
3977  // xoris r0,r3,0x1234
3978  // cmplwi cr0,r0,0x5678
3979  // beq cr0,L6
3980  SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS,
3981  getI32Imm(Imm >> 16, dl)), 0);
3982  return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor,
3983  getI32Imm(Imm & 0xFFFF, dl)), 0);
3984  }
3985  Opc = PPC::CMPLW;
3986  } else if (ISD::isUnsignedIntSetCC(CC)) {
3987  if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm))
3988  return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3989  getI32Imm(Imm & 0xFFFF, dl)), 0);
3990  Opc = PPC::CMPLW;
3991  } else {
3992  int16_t SImm;
3993  if (isIntS16Immediate(RHS, SImm))
3994  return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3995  getI32Imm((int)SImm & 0xFFFF,
3996  dl)),
3997  0);
3998  Opc = PPC::CMPW;
3999  }
4000  } else if (LHS.getValueType() == MVT::i64) {
4001  uint64_t Imm;
4002  if (CC == ISD::SETEQ || CC == ISD::SETNE) {
4003  if (isInt64Immediate(RHS.getNode(), Imm)) {
4004  // SETEQ/SETNE comparison with 16-bit immediate, fold it.
4005  if (isUInt<16>(Imm))
4006  return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
4007  getI32Imm(Imm & 0xFFFF, dl)),
4008  0);
4009  // If this is a 16-bit signed immediate, fold it.
4010  if (isInt<16>(Imm))
4011  return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
4012  getI32Imm(Imm & 0xFFFF, dl)),
4013  0);
4014 
4015  // For non-equality comparisons, the default code would materialize the
4016  // constant, then compare against it, like this:
4017  // lis r2, 4660
4018  // ori r2, r2, 22136
4019  // cmpd cr0, r3, r2
4020  // Since we are just comparing for equality, we can emit this instead:
4021  // xoris r0,r3,0x1234
4022  // cmpldi cr0,r0,0x5678
4023  // beq cr0,L6
4024  if (isUInt<32>(Imm)) {
4025  SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS,
4026  getI64Imm(Imm >> 16, dl)), 0);
4027  return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor,
4028  getI64Imm(Imm & 0xFFFF, dl)),
4029  0);
4030  }
4031  }
4032  Opc = PPC::CMPLD;
4033  } else if (ISD::isUnsignedIntSetCC(CC)) {
4034  if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm))
4035  return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
4036  getI64Imm(Imm & 0xFFFF, dl)), 0);
4037  Opc = PPC::CMPLD;
4038  } else {
4039  int16_t SImm;
4040  if (isIntS16Immediate(RHS, SImm))
4041  return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
4042  getI64Imm(SImm & 0xFFFF, dl)),
4043  0);
4044  Opc = PPC::CMPD;
4045  }
4046  } else if (LHS.getValueType() == MVT::f32) {
4047  if (Subtarget->hasSPE()) {
4048  switch (CC) {
4049  default:
4050  case ISD::SETEQ:
4051  case ISD::SETNE:
4052  Opc = PPC::EFSCMPEQ;
4053  break;
4054  case ISD::SETLT:
4055  case ISD::SETGE:
4056  case ISD::SETOLT:
4057  case ISD::SETOGE:
4058  case ISD::SETULT:
4059  case ISD::SETUGE:
4060  Opc = PPC::EFSCMPLT;
4061  break;
4062  case ISD::SETGT:
4063  case ISD::SETLE:
4064  case ISD::SETOGT:
4065  case ISD::SETOLE:
4066  case ISD::SETUGT:
4067  case ISD::SETULE:
4068  Opc = PPC::EFSCMPGT;
4069  break;
4070  }
4071  } else
4072  Opc = PPC::FCMPUS;
4073  } else if (LHS.getValueType() == MVT::f64) {
4074  if (Subtarget->hasSPE()) {
4075  switch (CC) {
4076  default:
4077  case ISD::SETEQ:
4078  case ISD::SETNE:
4079  Opc = PPC::EFDCMPEQ;
4080  break;
4081  case ISD::SETLT:
4082  case ISD::SETGE:
4083  case ISD::SETOLT:
4084  case ISD::SETOGE:
4085  case ISD::SETULT:
4086  case ISD::SETUGE:
4087  Opc = PPC::EFDCMPLT;
4088  break;
4089  case ISD::SETGT:
4090  case ISD::SETLE:
4091  case ISD::SETOGT:
4092  case ISD::SETOLE:
4093  case ISD::SETUGT:
4094  case ISD::SETULE:
4095  Opc = PPC::EFDCMPGT;
4096  break;
4097  }
4098  } else
4099  Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
4100  } else {
4101  assert(LHS.getValueType() == MVT::f128 && "Unknown vt!");
4102  assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector");
4103  Opc = PPC::XSCMPUQP;
4104  }
4105  if (Chain)
4106  return SDValue(
4107  CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain),
4108  0);
4109  else
4110  return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0);
4111 }
4112 
4114  const PPCSubtarget *Subtarget) {
4115  // For SPE instructions, the result is in GT bit of the CR
4116  bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint();
4117 
4118  switch (CC) {
4119  case ISD::SETUEQ:
4120  case ISD::SETONE:
4121  case ISD::SETOLE:
4122  case ISD::SETOGE:
4123  llvm_unreachable("Should be lowered by legalize!");
4124  default: llvm_unreachable("Unknown condition!");
4125  case ISD::SETOEQ:
4126  case ISD::SETEQ:
4127  return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ;
4128  case ISD::SETUNE:
4129  case ISD::SETNE:
4130  return UseSPE ? PPC::PRED_LE : PPC::PRED_NE;
4131  case ISD::SETOLT:
4132  case ISD::SETLT:
4133  return UseSPE ? PPC::PRED_GT : PPC::PRED_LT;
4134  case ISD::SETULE:
4135  case ISD::SETLE:
4136  return PPC::PRED_LE;
4137  case ISD::SETOGT:
4138  case ISD::SETGT:
4139  return PPC::PRED_GT;
4140  case ISD::SETUGE:
4141  case ISD::SETGE:
4142  return UseSPE ? PPC::PRED_LE : PPC::PRED_GE;
4143  case ISD::SETO: return PPC::PRED_NU;
4144  case ISD::SETUO: return PPC::PRED_UN;
4145  // These two are invalid for floating point. Assume we have int.
4146  case ISD::SETULT: return PPC::PRED_LT;
4147  case ISD::SETUGT: return PPC::PRED_GT;
4148  }
4149 }
4150 
4151 /// getCRIdxForSetCC - Return the index of the condition register field
4152 /// associated with the SetCC condition, and whether or not the field is
4153 /// treated as inverted. That is, lt = 0; ge = 0 inverted.
4154 static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) {
4155  Invert = false;
4156  switch (CC) {
4157  default: llvm_unreachable("Unknown condition!");
4158  case ISD::SETOLT:
4159  case ISD::SETLT: return 0; // Bit #0 = SETOLT
4160  case ISD::SETOGT:
4161  case ISD::SETGT: return 1; // Bit #1 = SETOGT
4162  case ISD::SETOEQ:
4163  case ISD::SETEQ: return 2; // Bit #2 = SETOEQ
4164  case ISD::SETUO: return 3; // Bit #3 = SETUO
4165  case ISD::SETUGE:
4166  case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE
4167  case ISD::SETULE:
4168  case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE
4169  case ISD::SETUNE:
4170  case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE
4171  case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO
4172  case ISD::SETUEQ:
4173  case ISD::SETOGE:
4174  case ISD::SETOLE:
4175  case ISD::SETONE:
4176  llvm_unreachable("Invalid branch code: should be expanded by legalize");
4177  // These are invalid for floating point. Assume integer.
4178  case ISD::SETULT: return 0;
4179  case ISD::SETUGT: return 1;
4180  }
4181 }
4182 
4183 // getVCmpInst: return the vector compare instruction for the specified
4184 // vector type and condition code. Since this is for altivec specific code,
4185 // only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128,
4186 // and v4f32).
4187 static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC,
4188  bool HasVSX, bool &Swap, bool &Negate) {
4189  Swap = false;
4190  Negate = false;
4191 
4192  if (VecVT.isFloatingPoint()) {
4193  /* Handle some cases by swapping input operands. */
4194  switch (CC) {
4195  case ISD::SETLE: CC = ISD::SETGE; Swap = true; break;
4196  case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
4197  case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break;
4198  case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break;
4199  case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
4200  case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break;
4201  default: break;
4202  }
4203  /* Handle some cases by negating the result. */
4204  switch (CC) {
4205  case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
4206  case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break;
4207  case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break;
4208  case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break;
4209  default: break;
4210  }
4211  /* We have instructions implementing the remaining cases. */
4212  switch (CC) {
4213  case ISD::SETEQ:
4214  case ISD::SETOEQ:
4215  if (VecVT == MVT::v4f32)
4216  return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP;
4217  else if (VecVT == MVT::v2f64)
4218  return PPC::XVCMPEQDP;
4219  break;
4220  case ISD::SETGT:
4221  case ISD::SETOGT:
4222  if (VecVT == MVT::v4f32)
4223  return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP;
4224  else if (VecVT == MVT::v2f64)
4225  return PPC::XVCMPGTDP;
4226  break;
4227  case ISD::SETGE:
4228  case ISD::SETOGE:
4229  if (VecVT == MVT::v4f32)
4230  return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP;
4231  else if (VecVT == MVT::v2f64)
4232  return PPC::XVCMPGEDP;
4233  break;
4234  default:
4235  break;
4236  }
4237  llvm_unreachable("Invalid floating-point vector compare condition");
4238  } else {
4239  /* Handle some cases by swapping input operands. */
4240  switch (CC) {
4241  case ISD::SETGE: CC = ISD::SETLE; Swap = true; break;
4242  case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
4243  case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
4244  case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break;
4245  default: break;
4246  }
4247  /* Handle some cases by negating the result. */
4248  switch (CC) {
4249  case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
4250  case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break;
4251  case ISD::SETLE: CC = ISD::SETGT; Negate = true; break;
4252  case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break;
4253  default: break;
4254  }
4255  /* We have instructions implementing the remaining cases. */
4256  switch (CC) {
4257  case ISD::SETEQ:
4258  case ISD::SETUEQ:
4259  if (VecVT == MVT::v16i8)
4260  return PPC::VCMPEQUB;
4261  else if (VecVT == MVT::v8i16)
4262  return PPC::VCMPEQUH;
4263  else if (VecVT == MVT::v4i32)
4264  return PPC::VCMPEQUW;
4265  else if (VecVT == MVT::v2i64)
4266  return PPC::VCMPEQUD;
4267  else if (VecVT == MVT::v1i128)
4268  return PPC::VCMPEQUQ;
4269  break;
4270  case ISD::SETGT:
4271  if (VecVT == MVT::v16i8)
4272  return PPC::VCMPGTSB;
4273  else if (VecVT == MVT::v8i16)
4274  return PPC::VCMPGTSH;
4275  else if (VecVT == MVT::v4i32)
4276  return PPC::VCMPGTSW;
4277  else if (VecVT == MVT::v2i64)
4278  return PPC::VCMPGTSD;
4279  else if (VecVT == MVT::v1i128)
4280  return PPC::VCMPGTSQ;
4281  break;
4282  case ISD::SETUGT:
4283  if (VecVT == MVT::v16i8)
4284  return PPC::VCMPGTUB;
4285  else if (VecVT == MVT::v8i16)
4286  return PPC::VCMPGTUH;
4287  else if (VecVT == MVT::v4i32)
4288  return PPC::VCMPGTUW;
4289  else if (VecVT == MVT::v2i64)
4290  return PPC::VCMPGTUD;
4291  else if (VecVT == MVT::v1i128)
4292  return PPC::VCMPGTUQ;
4293  break;
4294  default:
4295  break;
4296  }
4297  llvm_unreachable("Invalid integer vector compare condition");
4298  }
4299 }
4300 
4301 bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
4302  SDLoc dl(N);
4303  unsigned Imm;
4304  bool IsStrict = N->isStrictFPOpcode();
4305  ISD::CondCode CC =
4306  cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
4307  EVT PtrVT =
4308  CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
4309  bool isPPC64 = (PtrVT == MVT::i64);
4310  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4311 
4312  SDValue LHS = N->getOperand(IsStrict ? 1 : 0);
4313  SDValue RHS = N->getOperand(IsStrict ? 2 : 1);
4314 
4315  if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) {
4316  // We can codegen setcc op, imm very efficiently compared to a brcond.
4317  // Check for those cases here.
4318  // setcc op, 0
4319  if (Imm == 0) {
4320  SDValue Op = LHS;
4321  switch (CC) {
4322  default: break;
4323  case ISD::SETEQ: {
4324  Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0);
4325  SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl),
4326  getI32Imm(31, dl) };
4327  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4328  return true;
4329  }
4330  case ISD::SETNE: {
4331  if (isPPC64) break;
4332  SDValue AD =
4333  SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4334  Op, getI32Imm(~0U, dl)), 0);
4335  CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1));
4336  return true;
4337  }
4338  case ISD::SETLT: {
4339  SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4340  getI32Imm(31, dl) };
4341  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4342  return true;
4343  }
4344  case ISD::SETGT: {
4345  SDValue T =
4346  SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0);
4347  T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0);
4348  SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl),
4349  getI32Imm(31, dl) };
4350  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4351  return true;
4352  }
4353  }
4354  } else if (Imm == ~0U) { // setcc op, -1
4355  SDValue Op = LHS;
4356  switch (CC) {
4357  default: break;
4358  case ISD::SETEQ:
4359  if (isPPC64) break;
4360  Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4361  Op, getI32Imm(1, dl)), 0);
4362  CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32,
4363  SDValue(CurDAG->getMachineNode(PPC::LI, dl,
4364  MVT::i32,
4365  getI32Imm(0, dl)),
4366  0), Op.getValue(1));
4367  return true;
4368  case ISD::SETNE: {
4369  if (isPPC64) break;
4370  Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0);
4371  SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4372  Op, getI32Imm(~0U, dl));
4373  CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op,
4374  SDValue(AD, 1));
4375  return true;
4376  }
4377  case ISD::SETLT: {
4378  SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op,
4379  getI32Imm(1, dl)), 0);
4380  SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD,
4381  Op), 0);
4382  SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
4383  getI32Imm(31, dl) };
4384  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4385  return true;
4386  }
4387  case ISD::SETGT: {
4388  SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4389  getI32Imm(31, dl) };
4390  Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4391  CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl));
4392  return true;
4393  }
4394  }
4395  }
4396  }
4397 
4398  // Altivec Vector compare instructions do not set any CR register by default and
4399  // vector compare operations return the same type as the operands.
4400  if (!IsStrict && LHS.getValueType().isVector()) {
4401  if (Subtarget->hasSPE())
4402  return false;
4403 
4404  EVT VecVT = LHS.getValueType();
4405  bool Swap, Negate;
4406  unsigned int VCmpInst =
4407  getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate);
4408  if (Swap)
4409  std::swap(LHS, RHS);
4410 
4411  EVT ResVT = VecVT.changeVectorElementTypeToInteger();
4412  if (Negate) {
4413  SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0);
4414  CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
4415  ResVT, VCmp, VCmp);
4416  return true;
4417  }
4418 
4419  CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS);
4420  return true;
4421  }
4422 
4423  if (Subtarget->useCRBits())
4424  return false;
4425 
4426  bool Inv;
4427  unsigned Idx = getCRIdxForSetCC(CC, Inv);
4428  SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain);
4429  if (IsStrict)
4430  CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1));
4431  SDValue IntCR;
4432 
4433  // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that
4434  // The correct compare instruction is already set by SelectCC()
4435  if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) {
4436  Idx = 1;
4437  }
4438 
4439  // Force the ccreg into CR7.
4440  SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32);
4441 
4442  SDValue InFlag; // Null incoming flag value.
4443  CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg,
4444  InFlag).getValue(1);
4445 
4446  IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg,
4447  CCReg), 0);
4448 
4449  SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl),
4450  getI32Imm(31, dl), getI32Imm(31, dl) };
4451  if (!Inv) {
4452  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4453  return true;
4454  }
4455 
4456  // Get the specified bit.
4457  SDValue Tmp =
4458  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4459  CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl));
4460  return true;
4461 }
4462 
4463 /// Does this node represent a load/store node whose address can be represented
4464 /// with a register plus an immediate that's a multiple of \p Val:
4465 bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const {
4466  LoadSDNode *LDN = dyn_cast<LoadSDNode>(N);
4467  StoreSDNode *STN = dyn_cast<StoreSDNode>(N);
4468  MemIntrinsicSDNode *MIN = dyn_cast<MemIntrinsicSDNode>(N);
4469  SDValue AddrOp;
4470  if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT))
4471  AddrOp = N->getOperand(1);
4472  else if (STN)
4473  AddrOp = STN->getOperand(2);
4474 
4475  // If the address points a frame object or a frame object with an offset,
4476  // we need to check the object alignment.
4477  short Imm = 0;
4478  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(
4479  AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) :
4480  AddrOp)) {
4481  // If op0 is a frame index that is under aligned, we can't do it either,
4482  // because it is translated to r31 or r1 + slot + offset. We won't know the
4483  // slot number until the stack frame is finalized.
4484  const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo();
4485  unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value();
4486  if ((SlotAlign % Val) != 0)
4487  return false;
4488 
4489  // If we have an offset, we need further check on the offset.
4490  if (AddrOp.getOpcode() != ISD::ADD)
4491  return true;
4492  }
4493 
4494  if (AddrOp.getOpcode() == ISD::ADD)
4495  return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val);
4496 
4497  // If the address comes from the outside, the offset will be zero.
4498  return AddrOp.getOpcode() == ISD::CopyFromReg;
4499 }
4500 
4501 void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
4502  // Transfer memoperands.
4503  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
4504  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
4505 }
4506 
4507 static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG,
4508  bool &NeedSwapOps, bool &IsUnCmp) {
4509 
4510  assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here.");
4511 
4512  SDValue LHS = N->getOperand(0);
4513  SDValue RHS = N->getOperand(1);
4514  SDValue TrueRes = N->getOperand(2);
4515  SDValue FalseRes = N->getOperand(3);
4516  ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(TrueRes);
4517  if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 &&
4518  N->getSimpleValueType(0) != MVT::i32))
4519  return false;
4520 
4521  // We are looking for any of:
4522  // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4523  // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4524  // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq)
4525  // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq)
4526  int64_t TrueResVal = TrueConst->getSExtValue();
4527  if ((TrueResVal < -1 || TrueResVal > 1) ||
4528  (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) ||
4529  (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) ||
4530  (TrueResVal == 0 &&
4531  (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ)))
4532  return false;
4533 
4534  SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC
4535  ? FalseRes
4536  : FalseRes.getOperand(0);
4537  bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC;
4538  if (SetOrSelCC.getOpcode() != ISD::SETCC &&
4539  SetOrSelCC.getOpcode() != ISD::SELECT_CC)
4540  return false;
4541 
4542  // Without this setb optimization, the outer SELECT_CC will be manually
4543  // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass
4544  // transforms pseudo instruction to isel instruction. When there are more than
4545  // one use for result like zext/sext, with current optimization we only see
4546  // isel is replaced by setb but can't see any significant gain. Since
4547  // setb has longer latency than original isel, we should avoid this. Another
4548  // point is that setb requires comparison always kept, it can break the
4549  // opportunity to get the comparison away if we have in future.
4550  if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse()))
4551  return false;
4552 
4553  SDValue InnerLHS = SetOrSelCC.getOperand(0);
4554  SDValue InnerRHS = SetOrSelCC.getOperand(1);
4555  ISD::CondCode InnerCC =
4556  cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get();
4557  // If the inner comparison is a select_cc, make sure the true/false values are
4558  // 1/-1 and canonicalize it if needed.
4559  if (InnerIsSel) {
4560  ConstantSDNode *SelCCTrueConst =
4561  dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(2));
4562  ConstantSDNode *SelCCFalseConst =
4563  dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(3));
4564  if (!SelCCTrueConst || !SelCCFalseConst)
4565  return false;
4566  int64_t SelCCTVal = SelCCTrueConst->getSExtValue();
4567  int64_t SelCCFVal = SelCCFalseConst->getSExtValue();
4568  // The values must be -1/1 (requiring a swap) or 1/-1.
4569  if (SelCCTVal == -1 && SelCCFVal == 1) {
4570  std::swap(InnerLHS, InnerRHS);
4571  } else if (SelCCTVal != 1 || SelCCFVal != -1)
4572  return false;
4573  }
4574 
4575  // Canonicalize unsigned case
4576  if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) {
4577  IsUnCmp = true;
4578  InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT;
4579  }
4580 
4581  bool InnerSwapped = false;
4582  if (LHS == InnerRHS && RHS == InnerLHS)
4583  InnerSwapped = true;
4584  else if (LHS != InnerLHS || RHS != InnerRHS)
4585  return false;
4586 
4587  switch (CC) {
4588  // (select_cc lhs, rhs, 0, \
4589  // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq)
4590  case ISD::SETEQ:
4591  if (!InnerIsSel)
4592  return false;
4593  if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT)
4594  return false;
4595  NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped;
4596  break;
4597 
4598  // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4599  // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt)
4600  // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt)
4601  // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4602  // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt)
4603  // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt)
4604  case ISD::SETULT:
4605  if (!IsUnCmp && InnerCC != ISD::SETNE)
4606  return false;
4607  IsUnCmp = true;
4608  [[fallthrough]];
4609  case ISD::SETLT:
4610  if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) ||
4611  (InnerCC == ISD::SETLT && InnerSwapped))
4612  NeedSwapOps = (TrueResVal == 1);
4613  else
4614  return false;
4615  break;
4616 
4617  // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4618  // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt)
4619  // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt)
4620  // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4621  // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt)
4622  // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt)
4623  case ISD::SETUGT:
4624  if (!IsUnCmp && InnerCC != ISD::SETNE)
4625  return false;
4626  IsUnCmp = true;
4627  [[fallthrough]];
4628  case ISD::SETGT:
4629  if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) ||
4630  (InnerCC == ISD::SETGT && InnerSwapped))
4631  NeedSwapOps = (TrueResVal == -1);
4632  else
4633  return false;
4634  break;
4635 
4636  default:
4637  return false;
4638  }
4639 
4640  LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: ");
4641  LLVM_DEBUG(N->dump());
4642 
4643  return true;
4644 }
4645 
4646 // Return true if it's a software square-root/divide operand.
4647 static bool isSWTestOp(SDValue N) {
4648  if (N.getOpcode() == PPCISD::FTSQRT)
4649  return true;
4650  if (N.getNumOperands() < 1 || !isa<ConstantSDNode>(N.getOperand(0)) ||
4651  N.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
4652  return false;
4653  switch (N.getConstantOperandVal(0)) {
4654  case Intrinsic::ppc_vsx_xvtdivdp:
4655  case Intrinsic::ppc_vsx_xvtdivsp:
4656  case Intrinsic::ppc_vsx_xvtsqrtdp:
4657  case Intrinsic::ppc_vsx_xvtsqrtsp:
4658  return true;
4659  }
4660  return false;
4661 }
4662 
4663 bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) {
4664  assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected.");
4665  // We are looking for following patterns, where `truncate to i1` actually has
4666  // the same semantic with `and 1`.
4667  // (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp)
4668  // (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp)
4669  // (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp)
4670  // (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp)
4671  // (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp)
4672  // (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp)
4673  // (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp)
4674  // (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp)
4675  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
4676  if (CC != ISD::SETEQ && CC != ISD::SETNE)
4677  return false;
4678 
4679  SDValue CmpRHS = N->getOperand(3);
4680  if (!isa<ConstantSDNode>(CmpRHS) ||
4681  cast<ConstantSDNode>(CmpRHS)->getSExtValue() != 0)
4682  return false;
4683 
4684  SDValue CmpLHS = N->getOperand(2);
4685  if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(CmpLHS.getOperand(0)))
4686  return false;
4687 
4688  unsigned PCC = 0;
4689  bool IsCCNE = CC == ISD::SETNE;
4690  if (CmpLHS.getOpcode() == ISD::AND &&
4691  isa<ConstantSDNode>(CmpLHS.getOperand(1)))
4692  switch (CmpLHS.getConstantOperandVal(1)) {
4693  case 1:
4694  PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4695  break;
4696  case 2:
4697  PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE;
4698  break;
4699  case 4:
4700  PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE;
4701  break;
4702  case 8:
4703  PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE;
4704  break;
4705  default:
4706  return false;
4707  }
4708  else if (CmpLHS.getOpcode() == ISD::TRUNCATE &&
4709  CmpLHS.getValueType() == MVT::i1)
4710  PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4711 
4712  if (PCC) {
4713  SDLoc dl(N);
4714  SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4),
4715  N->getOperand(0)};
4716  CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
4717  return true;
4718  }
4719  return false;
4720 }
4721 
4722 bool PPCDAGToDAGISel::trySelectLoopCountIntrinsic(SDNode *N) {
4723  // Sometimes the promoted value of the intrinsic is ANDed by some non-zero
4724  // value, for example when crbits is disabled. If so, select the
4725  // loop_decrement intrinsics now.
4726  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
4727  SDValue LHS = N->getOperand(2), RHS = N->getOperand(3);
4728 
4729  if (LHS.getOpcode() != ISD::AND || !isa<ConstantSDNode>(LHS.getOperand(1)) ||
4730  isNullConstant(LHS.getOperand(1)))
4731  return false;
4732 
4733  if (LHS.getOperand(0).getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4734  cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() !=
4735  Intrinsic::loop_decrement)
4736  return false;
4737 
4738  if (!isa<ConstantSDNode>(RHS))
4739  return false;
4740 
4741  assert((CC == ISD::SETEQ || CC == ISD::SETNE) &&
4742  "Counter decrement comparison is not EQ or NE");
4743 
4744  SDValue OldDecrement = LHS.getOperand(0);
4745  assert(OldDecrement.hasOneUse() && "loop decrement has more than one use!");
4746