Bug Summary

File:lib/Target/PowerPC/PPCISelDAGToDAG.cpp
Warning:line 5328, column 11
Array subscript is undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name PPCISelDAGToDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/lib/Target/PowerPC -I /build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC -I /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn359999/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/lib/Target/PowerPC -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn359999=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-05-06-051613-19774-1 -x c++ /build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp -faddrsig
1//===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines a pattern matching instruction selector for PowerPC,
10// converting from a legalized dag to a PPC dag.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/PPCMCTargetDesc.h"
15#include "MCTargetDesc/PPCPredicates.h"
16#include "PPC.h"
17#include "PPCISelLowering.h"
18#include "PPCMachineFunctionInfo.h"
19#include "PPCSubtarget.h"
20#include "PPCTargetMachine.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/Analysis/BranchProbabilityInfo.h"
28#include "llvm/CodeGen/FunctionLoweringInfo.h"
29#include "llvm/CodeGen/ISDOpcodes.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/SelectionDAG.h"
35#include "llvm/CodeGen/SelectionDAGISel.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/TargetInstrInfo.h"
38#include "llvm/CodeGen/TargetRegisterInfo.h"
39#include "llvm/CodeGen/ValueTypes.h"
40#include "llvm/IR/BasicBlock.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/Function.h"
43#include "llvm/IR/GlobalValue.h"
44#include "llvm/IR/InlineAsm.h"
45#include "llvm/IR/InstrTypes.h"
46#include "llvm/IR/Module.h"
47#include "llvm/Support/Casting.h"
48#include "llvm/Support/CodeGen.h"
49#include "llvm/Support/CommandLine.h"
50#include "llvm/Support/Compiler.h"
51#include "llvm/Support/Debug.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/KnownBits.h"
54#include "llvm/Support/MachineValueType.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/raw_ostream.h"
57#include <algorithm>
58#include <cassert>
59#include <cstdint>
60#include <iterator>
61#include <limits>
62#include <memory>
63#include <new>
64#include <tuple>
65#include <utility>
66
67using namespace llvm;
68
69#define DEBUG_TYPE"ppc-codegen" "ppc-codegen"
70
71STATISTIC(NumSextSetcc,static llvm::Statistic NumSextSetcc = {"ppc-codegen", "NumSextSetcc"
, "Number of (sext(setcc)) nodes expanded into GPR sequence."
, {0}, {false}}
72 "Number of (sext(setcc)) nodes expanded into GPR sequence.")static llvm::Statistic NumSextSetcc = {"ppc-codegen", "NumSextSetcc"
, "Number of (sext(setcc)) nodes expanded into GPR sequence."
, {0}, {false}}
;
73STATISTIC(NumZextSetcc,static llvm::Statistic NumZextSetcc = {"ppc-codegen", "NumZextSetcc"
, "Number of (zext(setcc)) nodes expanded into GPR sequence."
, {0}, {false}}
74 "Number of (zext(setcc)) nodes expanded into GPR sequence.")static llvm::Statistic NumZextSetcc = {"ppc-codegen", "NumZextSetcc"
, "Number of (zext(setcc)) nodes expanded into GPR sequence."
, {0}, {false}}
;
75STATISTIC(SignExtensionsAdded,static llvm::Statistic SignExtensionsAdded = {"ppc-codegen", "SignExtensionsAdded"
, "Number of sign extensions for compare inputs added.", {0},
{false}}
76 "Number of sign extensions for compare inputs added.")static llvm::Statistic SignExtensionsAdded = {"ppc-codegen", "SignExtensionsAdded"
, "Number of sign extensions for compare inputs added.", {0},
{false}}
;
77STATISTIC(ZeroExtensionsAdded,static llvm::Statistic ZeroExtensionsAdded = {"ppc-codegen", "ZeroExtensionsAdded"
, "Number of zero extensions for compare inputs added.", {0},
{false}}
78 "Number of zero extensions for compare inputs added.")static llvm::Statistic ZeroExtensionsAdded = {"ppc-codegen", "ZeroExtensionsAdded"
, "Number of zero extensions for compare inputs added.", {0},
{false}}
;
79STATISTIC(NumLogicOpsOnComparison,static llvm::Statistic NumLogicOpsOnComparison = {"ppc-codegen"
, "NumLogicOpsOnComparison", "Number of logical ops on i1 values calculated in GPR."
, {0}, {false}}
80 "Number of logical ops on i1 values calculated in GPR.")static llvm::Statistic NumLogicOpsOnComparison = {"ppc-codegen"
, "NumLogicOpsOnComparison", "Number of logical ops on i1 values calculated in GPR."
, {0}, {false}}
;
81STATISTIC(OmittedForNonExtendUses,static llvm::Statistic OmittedForNonExtendUses = {"ppc-codegen"
, "OmittedForNonExtendUses", "Number of compares not eliminated as they have non-extending uses."
, {0}, {false}}
82 "Number of compares not eliminated as they have non-extending uses.")static llvm::Statistic OmittedForNonExtendUses = {"ppc-codegen"
, "OmittedForNonExtendUses", "Number of compares not eliminated as they have non-extending uses."
, {0}, {false}}
;
83STATISTIC(NumP9Setb,static llvm::Statistic NumP9Setb = {"ppc-codegen", "NumP9Setb"
, "Number of compares lowered to setb.", {0}, {false}}
84 "Number of compares lowered to setb.")static llvm::Statistic NumP9Setb = {"ppc-codegen", "NumP9Setb"
, "Number of compares lowered to setb.", {0}, {false}}
;
85
86// FIXME: Remove this once the bug has been fixed!
87cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug",
88cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden);
89
90static cl::opt<bool>
91 UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true),
92 cl::desc("use aggressive ppc isel for bit permutations"),
93 cl::Hidden);
94static cl::opt<bool> BPermRewriterNoMasking(
95 "ppc-bit-perm-rewriter-stress-rotates",
96 cl::desc("stress rotate selection in aggressive ppc isel for "
97 "bit permutations"),
98 cl::Hidden);
99
100static cl::opt<bool> EnableBranchHint(
101 "ppc-use-branch-hint", cl::init(true),
102 cl::desc("Enable static hinting of branches on ppc"),
103 cl::Hidden);
104
105static cl::opt<bool> EnableTLSOpt(
106 "ppc-tls-opt", cl::init(true),
107 cl::desc("Enable tls optimization peephole"),
108 cl::Hidden);
109
110enum ICmpInGPRType { ICGPR_All, ICGPR_None, ICGPR_I32, ICGPR_I64,
111 ICGPR_NonExtIn, ICGPR_Zext, ICGPR_Sext, ICGPR_ZextI32,
112 ICGPR_SextI32, ICGPR_ZextI64, ICGPR_SextI64 };
113
114static cl::opt<ICmpInGPRType> CmpInGPR(
115 "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All),
116 cl::desc("Specify the types of comparisons to emit GPR-only code for."),
117 cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons.")llvm::cl::OptionEnumValue { "none", int(ICGPR_None), "Do not modify integer comparisons."
}
,
118 clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs.")llvm::cl::OptionEnumValue { "all", int(ICGPR_All), "All possible int comparisons in GPRs."
}
,
119 clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs.")llvm::cl::OptionEnumValue { "i32", int(ICGPR_I32), "Only i32 comparisons in GPRs."
}
,
120 clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs.")llvm::cl::OptionEnumValue { "i64", int(ICGPR_I64), "Only i64 comparisons in GPRs."
}
,
121 clEnumValN(ICGPR_NonExtIn, "nonextin",llvm::cl::OptionEnumValue { "nonextin", int(ICGPR_NonExtIn), "Only comparisons where inputs don't need [sz]ext."
}
122 "Only comparisons where inputs don't need [sz]ext.")llvm::cl::OptionEnumValue { "nonextin", int(ICGPR_NonExtIn), "Only comparisons where inputs don't need [sz]ext."
}
,
123 clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result.")llvm::cl::OptionEnumValue { "zext", int(ICGPR_Zext), "Only comparisons with zext result."
}
,
124 clEnumValN(ICGPR_ZextI32, "zexti32",llvm::cl::OptionEnumValue { "zexti32", int(ICGPR_ZextI32), "Only i32 comparisons with zext result."
}
125 "Only i32 comparisons with zext result.")llvm::cl::OptionEnumValue { "zexti32", int(ICGPR_ZextI32), "Only i32 comparisons with zext result."
}
,
126 clEnumValN(ICGPR_ZextI64, "zexti64",llvm::cl::OptionEnumValue { "zexti64", int(ICGPR_ZextI64), "Only i64 comparisons with zext result."
}
127 "Only i64 comparisons with zext result.")llvm::cl::OptionEnumValue { "zexti64", int(ICGPR_ZextI64), "Only i64 comparisons with zext result."
}
,
128 clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result.")llvm::cl::OptionEnumValue { "sext", int(ICGPR_Sext), "Only comparisons with sext result."
}
,
129 clEnumValN(ICGPR_SextI32, "sexti32",llvm::cl::OptionEnumValue { "sexti32", int(ICGPR_SextI32), "Only i32 comparisons with sext result."
}
130 "Only i32 comparisons with sext result.")llvm::cl::OptionEnumValue { "sexti32", int(ICGPR_SextI32), "Only i32 comparisons with sext result."
}
,
131 clEnumValN(ICGPR_SextI64, "sexti64",llvm::cl::OptionEnumValue { "sexti64", int(ICGPR_SextI64), "Only i64 comparisons with sext result."
}
132 "Only i64 comparisons with sext result.")llvm::cl::OptionEnumValue { "sexti64", int(ICGPR_SextI64), "Only i64 comparisons with sext result."
}
));
133namespace {
134
135 //===--------------------------------------------------------------------===//
136 /// PPCDAGToDAGISel - PPC specific code to select PPC machine
137 /// instructions for SelectionDAG operations.
138 ///
139 class PPCDAGToDAGISel : public SelectionDAGISel {
140 const PPCTargetMachine &TM;
141 const PPCSubtarget *PPCSubTarget;
142 const PPCTargetLowering *PPCLowering;
143 unsigned GlobalBaseReg;
144
145 public:
146 explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel)
147 : SelectionDAGISel(tm, OptLevel), TM(tm) {}
148
149 bool runOnMachineFunction(MachineFunction &MF) override {
150 // Make sure we re-emit a set of the global base reg if necessary
151 GlobalBaseReg = 0;
152 PPCSubTarget = &MF.getSubtarget<PPCSubtarget>();
153 PPCLowering = PPCSubTarget->getTargetLowering();
154 SelectionDAGISel::runOnMachineFunction(MF);
155
156 if (!PPCSubTarget->isSVR4ABI())
157 InsertVRSaveCode(MF);
158
159 return true;
160 }
161
162 void PreprocessISelDAG() override;
163 void PostprocessISelDAG() override;
164
165 /// getI16Imm - Return a target constant with the specified value, of type
166 /// i16.
167 inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) {
168 return CurDAG->getTargetConstant(Imm, dl, MVT::i16);
169 }
170
171 /// getI32Imm - Return a target constant with the specified value, of type
172 /// i32.
173 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
174 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
175 }
176
177 /// getI64Imm - Return a target constant with the specified value, of type
178 /// i64.
179 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) {
180 return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
181 }
182
183 /// getSmallIPtrImm - Return a target constant of pointer type.
184 inline SDValue getSmallIPtrImm(unsigned Imm, const SDLoc &dl) {
185 return CurDAG->getTargetConstant(
186 Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
187 }
188
189 /// isRotateAndMask - Returns true if Mask and Shift can be folded into a
190 /// rotate and mask opcode and mask operation.
191 static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask,
192 unsigned &SH, unsigned &MB, unsigned &ME);
193
194 /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC
195 /// base register. Return the virtual register that holds this value.
196 SDNode *getGlobalBaseReg();
197
198 void selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset = 0);
199
200 // Select - Convert the specified operand from a target-independent to a
201 // target-specific node if it hasn't already been changed.
202 void Select(SDNode *N) override;
203
204 bool tryBitfieldInsert(SDNode *N);
205 bool tryBitPermutation(SDNode *N);
206 bool tryIntCompareInGPR(SDNode *N);
207
208 // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into
209 // an X-Form load instruction with the offset being a relocation coming from
210 // the PPCISD::ADD_TLS.
211 bool tryTLSXFormLoad(LoadSDNode *N);
212 // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into
213 // an X-Form store instruction with the offset being a relocation coming from
214 // the PPCISD::ADD_TLS.
215 bool tryTLSXFormStore(StoreSDNode *N);
216 /// SelectCC - Select a comparison of the specified values with the
217 /// specified condition code, returning the CR# of the expression.
218 SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
219 const SDLoc &dl);
220
221 /// SelectAddrImm - Returns true if the address N can be represented by
222 /// a base register plus a signed 16-bit displacement [r+imm].
223 bool SelectAddrImm(SDValue N, SDValue &Disp,
224 SDValue &Base) {
225 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 0);
226 }
227
228 /// SelectAddrImmOffs - Return true if the operand is valid for a preinc
229 /// immediate field. Note that the operand at this point is already the
230 /// result of a prior SelectAddressRegImm call.
231 bool SelectAddrImmOffs(SDValue N, SDValue &Out) const {
232 if (N.getOpcode() == ISD::TargetConstant ||
233 N.getOpcode() == ISD::TargetGlobalAddress) {
234 Out = N;
235 return true;
236 }
237
238 return false;
239 }
240
241 /// SelectAddrIdx - Given the specified addressed, check to see if it can be
242 /// represented as an indexed [r+r] operation. Returns false if it can
243 /// be represented by [r+imm], which are preferred.
244 bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
245 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG);
246 }
247
248 /// SelectAddrIdxOnly - Given the specified addressed, force it to be
249 /// represented as an indexed [r+r] operation.
250 bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
251 return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG);
252 }
253
254 /// SelectAddrImmX4 - Returns true if the address N can be represented by
255 /// a base register plus a signed 16-bit displacement that is a multiple of 4.
256 /// Suitable for use by STD and friends.
257 bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
258 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 4);
259 }
260
261 bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
262 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 16);
263 }
264
265 // Select an address into a single register.
266 bool SelectAddr(SDValue N, SDValue &Base) {
267 Base = N;
268 return true;
269 }
270
271 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
272 /// inline asm expressions. It is always correct to compute the value into
273 /// a register. The case of adding a (possibly relocatable) constant to a
274 /// register can be improved, but it is wrong to substitute Reg+Reg for
275 /// Reg in an asm, because the load or store opcode would have to change.
276 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
277 unsigned ConstraintID,
278 std::vector<SDValue> &OutOps) override {
279 switch(ConstraintID) {
280 default:
281 errs() << "ConstraintID: " << ConstraintID << "\n";
282 llvm_unreachable("Unexpected asm memory constraint")::llvm::llvm_unreachable_internal("Unexpected asm memory constraint"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 282)
;
283 case InlineAsm::Constraint_es:
284 case InlineAsm::Constraint_i:
285 case InlineAsm::Constraint_m:
286 case InlineAsm::Constraint_o:
287 case InlineAsm::Constraint_Q:
288 case InlineAsm::Constraint_Z:
289 case InlineAsm::Constraint_Zy:
290 // We need to make sure that this one operand does not end up in r0
291 // (because we might end up lowering this as 0(%op)).
292 const TargetRegisterInfo *TRI = PPCSubTarget->getRegisterInfo();
293 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1);
294 SDLoc dl(Op);
295 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
296 SDValue NewOp =
297 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
298 dl, Op.getValueType(),
299 Op, RC), 0);
300
301 OutOps.push_back(NewOp);
302 return false;
303 }
304 return true;
305 }
306
307 void InsertVRSaveCode(MachineFunction &MF);
308
309 StringRef getPassName() const override {
310 return "PowerPC DAG->DAG Pattern Instruction Selection";
311 }
312
313// Include the pieces autogenerated from the target description.
314#include "PPCGenDAGISel.inc"
315
316private:
317 bool trySETCC(SDNode *N);
318
319 void PeepholePPC64();
320 void PeepholePPC64ZExt();
321 void PeepholeCROps();
322
323 SDValue combineToCMPB(SDNode *N);
324 void foldBoolExts(SDValue &Res, SDNode *&N);
325
326 bool AllUsersSelectZero(SDNode *N);
327 void SwapAllSelectUsers(SDNode *N);
328
329 bool isOffsetMultipleOf(SDNode *N, unsigned Val) const;
330 void transferMemOperands(SDNode *N, SDNode *Result);
331 };
332
333} // end anonymous namespace
334
335/// InsertVRSaveCode - Once the entire function has been instruction selected,
336/// all virtual registers are created and all machine instructions are built,
337/// check to see if we need to save/restore VRSAVE. If so, do it.
338void PPCDAGToDAGISel::InsertVRSaveCode(MachineFunction &Fn) {
339 // Check to see if this function uses vector registers, which means we have to
340 // save and restore the VRSAVE register and update it with the regs we use.
341 //
342 // In this case, there will be virtual registers of vector type created
343 // by the scheduler. Detect them now.
344 bool HasVectorVReg = false;
345 for (unsigned i = 0, e = RegInfo->getNumVirtRegs(); i != e; ++i) {
346 unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
347 if (RegInfo->getRegClass(Reg) == &PPC::VRRCRegClass) {
348 HasVectorVReg = true;
349 break;
350 }
351 }
352 if (!HasVectorVReg) return; // nothing to do.
353
354 // If we have a vector register, we want to emit code into the entry and exit
355 // blocks to save and restore the VRSAVE register. We do this here (instead
356 // of marking all vector instructions as clobbering VRSAVE) for two reasons:
357 //
358 // 1. This (trivially) reduces the load on the register allocator, by not
359 // having to represent the live range of the VRSAVE register.
360 // 2. This (more significantly) allows us to create a temporary virtual
361 // register to hold the saved VRSAVE value, allowing this temporary to be
362 // register allocated, instead of forcing it to be spilled to the stack.
363
364 // Create two vregs - one to hold the VRSAVE register that is live-in to the
365 // function and one for the value after having bits or'd into it.
366 unsigned InVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
367 unsigned UpdatedVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
368
369 const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo();
370 MachineBasicBlock &EntryBB = *Fn.begin();
371 DebugLoc dl;
372 // Emit the following code into the entry block:
373 // InVRSAVE = MFVRSAVE
374 // UpdatedVRSAVE = UPDATE_VRSAVE InVRSAVE
375 // MTVRSAVE UpdatedVRSAVE
376 MachineBasicBlock::iterator IP = EntryBB.begin(); // Insert Point
377 BuildMI(EntryBB, IP, dl, TII.get(PPC::MFVRSAVE), InVRSAVE);
378 BuildMI(EntryBB, IP, dl, TII.get(PPC::UPDATE_VRSAVE),
379 UpdatedVRSAVE).addReg(InVRSAVE);
380 BuildMI(EntryBB, IP, dl, TII.get(PPC::MTVRSAVE)).addReg(UpdatedVRSAVE);
381
382 // Find all return blocks, outputting a restore in each epilog.
383 for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) {
384 if (BB->isReturnBlock()) {
385 IP = BB->end(); --IP;
386
387 // Skip over all terminator instructions, which are part of the return
388 // sequence.
389 MachineBasicBlock::iterator I2 = IP;
390 while (I2 != BB->begin() && (--I2)->isTerminator())
391 IP = I2;
392
393 // Emit: MTVRSAVE InVRSave
394 BuildMI(*BB, IP, dl, TII.get(PPC::MTVRSAVE)).addReg(InVRSAVE);
395 }
396 }
397}
398
399/// getGlobalBaseReg - Output the instructions required to put the
400/// base address to use for accessing globals into a register.
401///
402SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
403 if (!GlobalBaseReg) {
404 const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo();
405 // Insert the set of GlobalBaseReg into the first MBB of the function
406 MachineBasicBlock &FirstMBB = MF->front();
407 MachineBasicBlock::iterator MBBI = FirstMBB.begin();
408 const Module *M = MF->getFunction().getParent();
409 DebugLoc dl;
410
411 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) {
412 if (PPCSubTarget->isTargetELF()) {
413 GlobalBaseReg = PPC::R30;
414 if (M->getPICLevel() == PICLevel::SmallPIC) {
415 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR));
416 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
417 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
418 } else {
419 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
420 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
421 unsigned TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
422 BuildMI(FirstMBB, MBBI, dl,
423 TII.get(PPC::UpdateGBR), GlobalBaseReg)
424 .addReg(TempReg, RegState::Define).addReg(GlobalBaseReg);
425 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
426 }
427 } else {
428 GlobalBaseReg =
429 RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass);
430 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
431 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
432 }
433 } else {
434 // We must ensure that this sequence is dominated by the prologue.
435 // FIXME: This is a bit of a big hammer since we don't get the benefits
436 // of shrink-wrapping whenever we emit this instruction. Considering
437 // this is used in any function where we emit a jump table, this may be
438 // a significant limitation. We should consider inserting this in the
439 // block where it is used and then commoning this sequence up if it
440 // appears in multiple places.
441 // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of
442 // MovePCtoLR8.
443 MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true);
444 GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
445 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8));
446 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg);
447 }
448 }
449 return CurDAG->getRegister(GlobalBaseReg,
450 PPCLowering->getPointerTy(CurDAG->getDataLayout()))
451 .getNode();
452}
453
454/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
455/// operand. If so Imm will receive the 32-bit value.
456static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
457 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
458 Imm = cast<ConstantSDNode>(N)->getZExtValue();
459 return true;
460 }
461 return false;
462}
463
464/// isInt64Immediate - This method tests to see if the node is a 64-bit constant
465/// operand. If so Imm will receive the 64-bit value.
466static bool isInt64Immediate(SDNode *N, uint64_t &Imm) {
467 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) {
468 Imm = cast<ConstantSDNode>(N)->getZExtValue();
469 return true;
470 }
471 return false;
472}
473
474// isInt32Immediate - This method tests to see if a constant operand.
475// If so Imm will receive the 32 bit value.
476static bool isInt32Immediate(SDValue N, unsigned &Imm) {
477 return isInt32Immediate(N.getNode(), Imm);
478}
479
480/// isInt64Immediate - This method tests to see if the value is a 64-bit
481/// constant operand. If so Imm will receive the 64-bit value.
482static bool isInt64Immediate(SDValue N, uint64_t &Imm) {
483 return isInt64Immediate(N.getNode(), Imm);
484}
485
486static unsigned getBranchHint(unsigned PCC, FunctionLoweringInfo *FuncInfo,
487 const SDValue &DestMBB) {
488 assert(isa<BasicBlockSDNode>(DestMBB))((isa<BasicBlockSDNode>(DestMBB)) ? static_cast<void
> (0) : __assert_fail ("isa<BasicBlockSDNode>(DestMBB)"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 488, __PRETTY_FUNCTION__))
;
489
490 if (!FuncInfo->BPI) return PPC::BR_NO_HINT;
491
492 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
493 const Instruction *BBTerm = BB->getTerminator();
494
495 if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT;
496
497 const BasicBlock *TBB = BBTerm->getSuccessor(0);
498 const BasicBlock *FBB = BBTerm->getSuccessor(1);
499
500 auto TProb = FuncInfo->BPI->getEdgeProbability(BB, TBB);
501 auto FProb = FuncInfo->BPI->getEdgeProbability(BB, FBB);
502
503 // We only want to handle cases which are easy to predict at static time, e.g.
504 // C++ throw statement, that is very likely not taken, or calling never
505 // returned function, e.g. stdlib exit(). So we set Threshold to filter
506 // unwanted cases.
507 //
508 // Below is LLVM branch weight table, we only want to handle case 1, 2
509 //
510 // Case Taken:Nontaken Example
511 // 1. Unreachable 1048575:1 C++ throw, stdlib exit(),
512 // 2. Invoke-terminating 1:1048575
513 // 3. Coldblock 4:64 __builtin_expect
514 // 4. Loop Branch 124:4 For loop
515 // 5. PH/ZH/FPH 20:12
516 const uint32_t Threshold = 10000;
517
518 if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb))
519 return PPC::BR_NO_HINT;
520
521 LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Use branch hint for '" <<
FuncInfo->Fn->getName() << "::" << BB->
getName() << "'\n" << " -> " << TBB->
getName() << ": " << TProb << "\n" <<
" -> " << FBB->getName() << ": " << FProb
<< "\n"; } } while (false)
522 << "::" << BB->getName() << "'\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Use branch hint for '" <<
FuncInfo->Fn->getName() << "::" << BB->
getName() << "'\n" << " -> " << TBB->
getName() << ": " << TProb << "\n" <<
" -> " << FBB->getName() << ": " << FProb
<< "\n"; } } while (false)
523 << " -> " << TBB->getName() << ": " << TProb << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Use branch hint for '" <<
FuncInfo->Fn->getName() << "::" << BB->
getName() << "'\n" << " -> " << TBB->
getName() << ": " << TProb << "\n" <<
" -> " << FBB->getName() << ": " << FProb
<< "\n"; } } while (false)
524 << " -> " << FBB->getName() << ": " << FProb << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Use branch hint for '" <<
FuncInfo->Fn->getName() << "::" << BB->
getName() << "'\n" << " -> " << TBB->
getName() << ": " << TProb << "\n" <<
" -> " << FBB->getName() << ": " << FProb
<< "\n"; } } while (false)
;
525
526 const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB);
527
528 // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities,
529 // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock
530 if (BBDN->getBasicBlock()->getBasicBlock() != TBB)
531 std::swap(TProb, FProb);
532
533 return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT;
534}
535
536// isOpcWithIntImmediate - This method tests to see if the node is a specific
537// opcode and that it has a immediate integer right operand.
538// If so Imm will receive the 32 bit value.
539static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
540 return N->getOpcode() == Opc
541 && isInt32Immediate(N->getOperand(1).getNode(), Imm);
542}
543
544void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset) {
545 SDLoc dl(SN);
546 int FI = cast<FrameIndexSDNode>(N)->getIndex();
547 SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
548 unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8;
549 if (SN->hasOneUse())
550 CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI,
551 getSmallIPtrImm(Offset, dl));
552 else
553 ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI,
554 getSmallIPtrImm(Offset, dl)));
555}
556
557bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask,
558 bool isShiftMask, unsigned &SH,
559 unsigned &MB, unsigned &ME) {
560 // Don't even go down this path for i64, since different logic will be
561 // necessary for rldicl/rldicr/rldimi.
562 if (N->getValueType(0) != MVT::i32)
563 return false;
564
565 unsigned Shift = 32;
566 unsigned Indeterminant = ~0; // bit mask marking indeterminant results
567 unsigned Opcode = N->getOpcode();
568 if (N->getNumOperands() != 2 ||
569 !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31))
570 return false;
571
572 if (Opcode == ISD::SHL) {
573 // apply shift left to mask if it comes first
574 if (isShiftMask) Mask = Mask << Shift;
575 // determine which bits are made indeterminant by shift
576 Indeterminant = ~(0xFFFFFFFFu << Shift);
577 } else if (Opcode == ISD::SRL) {
578 // apply shift right to mask if it comes first
579 if (isShiftMask) Mask = Mask >> Shift;
580 // determine which bits are made indeterminant by shift
581 Indeterminant = ~(0xFFFFFFFFu >> Shift);
582 // adjust for the left rotate
583 Shift = 32 - Shift;
584 } else if (Opcode == ISD::ROTL) {
585 Indeterminant = 0;
586 } else {
587 return false;
588 }
589
590 // if the mask doesn't intersect any Indeterminant bits
591 if (Mask && !(Mask & Indeterminant)) {
592 SH = Shift & 31;
593 // make sure the mask is still a mask (wrap arounds may not be)
594 return isRunOfOnes(Mask, MB, ME);
595 }
596 return false;
597}
598
599bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) {
600 SDValue Base = ST->getBasePtr();
601 if (Base.getOpcode() != PPCISD::ADD_TLS)
602 return false;
603 SDValue Offset = ST->getOffset();
604 if (!Offset.isUndef())
605 return false;
606
607 SDLoc dl(ST);
608 EVT MemVT = ST->getMemoryVT();
609 EVT RegVT = ST->getValue().getValueType();
610
611 unsigned Opcode;
612 switch (MemVT.getSimpleVT().SimpleTy) {
613 default:
614 return false;
615 case MVT::i8: {
616 Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS;
617 break;
618 }
619 case MVT::i16: {
620 Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS;
621 break;
622 }
623 case MVT::i32: {
624 Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS;
625 break;
626 }
627 case MVT::i64: {
628 Opcode = PPC::STDXTLS;
629 break;
630 }
631 }
632 SDValue Chain = ST->getChain();
633 SDVTList VTs = ST->getVTList();
634 SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1),
635 Chain};
636 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
637 transferMemOperands(ST, MN);
638 ReplaceNode(ST, MN);
639 return true;
640}
641
642bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) {
643 SDValue Base = LD->getBasePtr();
644 if (Base.getOpcode() != PPCISD::ADD_TLS)
645 return false;
646 SDValue Offset = LD->getOffset();
647 if (!Offset.isUndef())
648 return false;
649
650 SDLoc dl(LD);
651 EVT MemVT = LD->getMemoryVT();
652 EVT RegVT = LD->getValueType(0);
653 unsigned Opcode;
654 switch (MemVT.getSimpleVT().SimpleTy) {
655 default:
656 return false;
657 case MVT::i8: {
658 Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS;
659 break;
660 }
661 case MVT::i16: {
662 Opcode = (RegVT == MVT::i32) ? PPC::LHZXTLS_32 : PPC::LHZXTLS;
663 break;
664 }
665 case MVT::i32: {
666 Opcode = (RegVT == MVT::i32) ? PPC::LWZXTLS_32 : PPC::LWZXTLS;
667 break;
668 }
669 case MVT::i64: {
670 Opcode = PPC::LDXTLS;
671 break;
672 }
673 }
674 SDValue Chain = LD->getChain();
675 SDVTList VTs = LD->getVTList();
676 SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain};
677 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
678 transferMemOperands(LD, MN);
679 ReplaceNode(LD, MN);
680 return true;
681}
682
683/// Turn an or of two masked values into the rotate left word immediate then
684/// mask insert (rlwimi) instruction.
685bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) {
686 SDValue Op0 = N->getOperand(0);
687 SDValue Op1 = N->getOperand(1);
688 SDLoc dl(N);
689
690 KnownBits LKnown = CurDAG->computeKnownBits(Op0);
691 KnownBits RKnown = CurDAG->computeKnownBits(Op1);
692
693 unsigned TargetMask = LKnown.Zero.getZExtValue();
694 unsigned InsertMask = RKnown.Zero.getZExtValue();
695
696 if ((TargetMask | InsertMask) == 0xFFFFFFFF) {
697 unsigned Op0Opc = Op0.getOpcode();
698 unsigned Op1Opc = Op1.getOpcode();
699 unsigned Value, SH = 0;
700 TargetMask = ~TargetMask;
701 InsertMask = ~InsertMask;
702
703 // If the LHS has a foldable shift and the RHS does not, then swap it to the
704 // RHS so that we can fold the shift into the insert.
705 if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) {
706 if (Op0.getOperand(0).getOpcode() == ISD::SHL ||
707 Op0.getOperand(0).getOpcode() == ISD::SRL) {
708 if (Op1.getOperand(0).getOpcode() != ISD::SHL &&
709 Op1.getOperand(0).getOpcode() != ISD::SRL) {
710 std::swap(Op0, Op1);
711 std::swap(Op0Opc, Op1Opc);
712 std::swap(TargetMask, InsertMask);
713 }
714 }
715 } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) {
716 if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL &&
717 Op1.getOperand(0).getOpcode() != ISD::SRL) {
718 std::swap(Op0, Op1);
719 std::swap(Op0Opc, Op1Opc);
720 std::swap(TargetMask, InsertMask);
721 }
722 }
723
724 unsigned MB, ME;
725 if (isRunOfOnes(InsertMask, MB, ME)) {
726 if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) &&
727 isInt32Immediate(Op1.getOperand(1), Value)) {
728 Op1 = Op1.getOperand(0);
729 SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value;
730 }
731 if (Op1Opc == ISD::AND) {
732 // The AND mask might not be a constant, and we need to make sure that
733 // if we're going to fold the masking with the insert, all bits not
734 // know to be zero in the mask are known to be one.
735 KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1));
736 bool CanFoldMask = InsertMask == MKnown.One.getZExtValue();
737
738 unsigned SHOpc = Op1.getOperand(0).getOpcode();
739 if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask &&
740 isInt32Immediate(Op1.getOperand(0).getOperand(1), Value)) {
741 // Note that Value must be in range here (less than 32) because
742 // otherwise there would not be any bits set in InsertMask.
743 Op1 = Op1.getOperand(0).getOperand(0);
744 SH = (SHOpc == ISD::SHL) ? Value : 32 - Value;
745 }
746 }
747
748 SH &= 31;
749 SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
750 getI32Imm(ME, dl) };
751 ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
752 return true;
753 }
754 }
755 return false;
756}
757
758// Predict the number of instructions that would be generated by calling
759// selectI64Imm(N).
760static unsigned selectI64ImmInstrCountDirect(int64_t Imm) {
761 // Assume no remaining bits.
762 unsigned Remainder = 0;
763 // Assume no shift required.
764 unsigned Shift = 0;
765
766 // If it can't be represented as a 32 bit value.
767 if (!isInt<32>(Imm)) {
768 Shift = countTrailingZeros<uint64_t>(Imm);
769 int64_t ImmSh = static_cast<uint64_t>(Imm) >> Shift;
770
771 // If the shifted value fits 32 bits.
772 if (isInt<32>(ImmSh)) {
773 // Go with the shifted value.
774 Imm = ImmSh;
775 } else {
776 // Still stuck with a 64 bit value.
777 Remainder = Imm;
778 Shift = 32;
779 Imm >>= 32;
780 }
781 }
782
783 // Intermediate operand.
784 unsigned Result = 0;
785
786 // Handle first 32 bits.
787 unsigned Lo = Imm & 0xFFFF;
788
789 // Simple value.
790 if (isInt<16>(Imm)) {
791 // Just the Lo bits.
792 ++Result;
793 } else if (Lo) {
794 // Handle the Hi bits and Lo bits.
795 Result += 2;
796 } else {
797 // Just the Hi bits.
798 ++Result;
799 }
800
801 // If no shift, we're done.
802 if (!Shift) return Result;
803
804 // If Hi word == Lo word,
805 // we can use rldimi to insert the Lo word into Hi word.
806 if ((unsigned)(Imm & 0xFFFFFFFF) == Remainder) {
807 ++Result;
808 return Result;
809 }
810
811 // Shift for next step if the upper 32-bits were not zero.
812 if (Imm)
813 ++Result;
814
815 // Add in the last bits as required.
816 if ((Remainder >> 16) & 0xFFFF)
817 ++Result;
818 if (Remainder & 0xFFFF)
819 ++Result;
820
821 return Result;
822}
823
824static uint64_t Rot64(uint64_t Imm, unsigned R) {
825 return (Imm << R) | (Imm >> (64 - R));
826}
827
828static unsigned selectI64ImmInstrCount(int64_t Imm) {
829 unsigned Count = selectI64ImmInstrCountDirect(Imm);
830
831 // If the instruction count is 1 or 2, we do not need further analysis
832 // since rotate + load constant requires at least 2 instructions.
833 if (Count <= 2)
834 return Count;
835
836 for (unsigned r = 1; r < 63; ++r) {
837 uint64_t RImm = Rot64(Imm, r);
838 unsigned RCount = selectI64ImmInstrCountDirect(RImm) + 1;
839 Count = std::min(Count, RCount);
840
841 // See comments in selectI64Imm for an explanation of the logic below.
842 unsigned LS = findLastSet(RImm);
843 if (LS != r-1)
844 continue;
845
846 uint64_t OnesMask = -(int64_t) (UINT64_C(1)1UL << (LS+1));
847 uint64_t RImmWithOnes = RImm | OnesMask;
848
849 RCount = selectI64ImmInstrCountDirect(RImmWithOnes) + 1;
850 Count = std::min(Count, RCount);
851 }
852
853 return Count;
854}
855
856// Select a 64-bit constant. For cost-modeling purposes, selectI64ImmInstrCount
857// (above) needs to be kept in sync with this function.
858static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl,
859 int64_t Imm) {
860 // Assume no remaining bits.
861 unsigned Remainder = 0;
862 // Assume no shift required.
863 unsigned Shift = 0;
864
865 // If it can't be represented as a 32 bit value.
866 if (!isInt<32>(Imm)) {
867 Shift = countTrailingZeros<uint64_t>(Imm);
868 int64_t ImmSh = static_cast<uint64_t>(Imm) >> Shift;
869
870 // If the shifted value fits 32 bits.
871 if (isInt<32>(ImmSh)) {
872 // Go with the shifted value.
873 Imm = ImmSh;
874 } else {
875 // Still stuck with a 64 bit value.
876 Remainder = Imm;
877 Shift = 32;
878 Imm >>= 32;
879 }
880 }
881
882 // Intermediate operand.
883 SDNode *Result;
884
885 // Handle first 32 bits.
886 unsigned Lo = Imm & 0xFFFF;
887 unsigned Hi = (Imm >> 16) & 0xFFFF;
888
889 auto getI32Imm = [CurDAG, dl](unsigned Imm) {
890 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
891 };
892
893 // Simple value.
894 if (isInt<16>(Imm)) {
895 uint64_t SextImm = SignExtend64(Lo, 16);
896 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
897 // Just the Lo bits.
898 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
899 } else if (Lo) {
900 // Handle the Hi bits.
901 unsigned OpC = Hi ? PPC::LIS8 : PPC::LI8;
902 Result = CurDAG->getMachineNode(OpC, dl, MVT::i64, getI32Imm(Hi));
903 // And Lo bits.
904 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64,
905 SDValue(Result, 0), getI32Imm(Lo));
906 } else {
907 // Just the Hi bits.
908 Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi));
909 }
910
911 // If no shift, we're done.
912 if (!Shift) return Result;
913
914 // If Hi word == Lo word,
915 // we can use rldimi to insert the Lo word into Hi word.
916 if ((unsigned)(Imm & 0xFFFFFFFF) == Remainder) {
917 SDValue Ops[] =
918 { SDValue(Result, 0), SDValue(Result, 0), getI32Imm(Shift), getI32Imm(0)};
919 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
920 }
921
922 // Shift for next step if the upper 32-bits were not zero.
923 if (Imm) {
924 Result = CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64,
925 SDValue(Result, 0),
926 getI32Imm(Shift),
927 getI32Imm(63 - Shift));
928 }
929
930 // Add in the last bits as required.
931 if ((Hi = (Remainder >> 16) & 0xFFFF)) {
932 Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64,
933 SDValue(Result, 0), getI32Imm(Hi));
934 }
935 if ((Lo = Remainder & 0xFFFF)) {
936 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64,
937 SDValue(Result, 0), getI32Imm(Lo));
938 }
939
940 return Result;
941}
942
943static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl,
944 int64_t Imm) {
945 unsigned Count = selectI64ImmInstrCountDirect(Imm);
946
947 // If the instruction count is 1 or 2, we do not need further analysis
948 // since rotate + load constant requires at least 2 instructions.
949 if (Count <= 2)
950 return selectI64ImmDirect(CurDAG, dl, Imm);
951
952 unsigned RMin = 0;
953
954 int64_t MatImm;
955 unsigned MaskEnd;
956
957 for (unsigned r = 1; r < 63; ++r) {
958 uint64_t RImm = Rot64(Imm, r);
959 unsigned RCount = selectI64ImmInstrCountDirect(RImm) + 1;
960 if (RCount < Count) {
961 Count = RCount;
962 RMin = r;
963 MatImm = RImm;
964 MaskEnd = 63;
965 }
966
967 // If the immediate to generate has many trailing zeros, it might be
968 // worthwhile to generate a rotated value with too many leading ones
969 // (because that's free with li/lis's sign-extension semantics), and then
970 // mask them off after rotation.
971
972 unsigned LS = findLastSet(RImm);
973 // We're adding (63-LS) higher-order ones, and we expect to mask them off
974 // after performing the inverse rotation by (64-r). So we need that:
975 // 63-LS == 64-r => LS == r-1
976 if (LS != r-1)
977 continue;
978
979 uint64_t OnesMask = -(int64_t) (UINT64_C(1)1UL << (LS+1));
980 uint64_t RImmWithOnes = RImm | OnesMask;
981
982 RCount = selectI64ImmInstrCountDirect(RImmWithOnes) + 1;
983 if (RCount < Count) {
984 Count = RCount;
985 RMin = r;
986 MatImm = RImmWithOnes;
987 MaskEnd = LS;
988 }
989 }
990
991 if (!RMin)
992 return selectI64ImmDirect(CurDAG, dl, Imm);
993
994 auto getI32Imm = [CurDAG, dl](unsigned Imm) {
995 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
996 };
997
998 SDValue Val = SDValue(selectI64ImmDirect(CurDAG, dl, MatImm), 0);
999 return CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Val,
1000 getI32Imm(64 - RMin), getI32Imm(MaskEnd));
1001}
1002
1003static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) {
1004 unsigned MaxTruncation = 0;
1005 // Cannot use range-based for loop here as we need the actual use (i.e. we
1006 // need the operand number corresponding to the use). A range-based for
1007 // will unbox the use and provide an SDNode*.
1008 for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end();
1009 Use != UseEnd; ++Use) {
1010 unsigned Opc =
1011 Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode();
1012 switch (Opc) {
1013 default: return 0;
1014 case ISD::TRUNCATE:
1015 if (Use->isMachineOpcode())
1016 return 0;
1017 MaxTruncation =
1018 std::max(MaxTruncation, Use->getValueType(0).getSizeInBits());
1019 continue;
1020 case ISD::STORE: {
1021 if (Use->isMachineOpcode())
1022 return 0;
1023 StoreSDNode *STN = cast<StoreSDNode>(*Use);
1024 unsigned MemVTSize = STN->getMemoryVT().getSizeInBits();
1025 if (MemVTSize == 64 || Use.getOperandNo() != 0)
1026 return 0;
1027 MaxTruncation = std::max(MaxTruncation, MemVTSize);
1028 continue;
1029 }
1030 case PPC::STW8:
1031 case PPC::STWX8:
1032 case PPC::STWU8:
1033 case PPC::STWUX8:
1034 if (Use.getOperandNo() != 0)
1035 return 0;
1036 MaxTruncation = std::max(MaxTruncation, 32u);
1037 continue;
1038 case PPC::STH8:
1039 case PPC::STHX8:
1040 case PPC::STHU8:
1041 case PPC::STHUX8:
1042 if (Use.getOperandNo() != 0)
1043 return 0;
1044 MaxTruncation = std::max(MaxTruncation, 16u);
1045 continue;
1046 case PPC::STB8:
1047 case PPC::STBX8:
1048 case PPC::STBU8:
1049 case PPC::STBUX8:
1050 if (Use.getOperandNo() != 0)
1051 return 0;
1052 MaxTruncation = std::max(MaxTruncation, 8u);
1053 continue;
1054 }
1055 }
1056 return MaxTruncation;
1057}
1058
1059// Select a 64-bit constant.
1060static SDNode *selectI64Imm(SelectionDAG *CurDAG, SDNode *N) {
1061 SDLoc dl(N);
1062
1063 // Get 64 bit value.
1064 int64_t Imm = cast<ConstantSDNode>(N)->getZExtValue();
1065 if (unsigned MinSize = allUsesTruncate(CurDAG, N)) {
1066 uint64_t SextImm = SignExtend64(Imm, MinSize);
1067 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
1068 if (isInt<16>(SextImm))
1069 return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
1070 }
1071 return selectI64Imm(CurDAG, dl, Imm);
1072}
1073
1074namespace {
1075
1076class BitPermutationSelector {
1077 struct ValueBit {
1078 SDValue V;
1079
1080 // The bit number in the value, using a convention where bit 0 is the
1081 // lowest-order bit.
1082 unsigned Idx;
1083
1084 // ConstZero means a bit we need to mask off.
1085 // Variable is a bit comes from an input variable.
1086 // VariableKnownToBeZero is also a bit comes from an input variable,
1087 // but it is known to be already zero. So we do not need to mask them.
1088 enum Kind {
1089 ConstZero,
1090 Variable,
1091 VariableKnownToBeZero
1092 } K;
1093
1094 ValueBit(SDValue V, unsigned I, Kind K = Variable)
1095 : V(V), Idx(I), K(K) {}
1096 ValueBit(Kind K = Variable)
1097 : V(SDValue(nullptr, 0)), Idx(UINT32_MAX(4294967295U)), K(K) {}
1098
1099 bool isZero() const {
1100 return K == ConstZero || K == VariableKnownToBeZero;
1101 }
1102
1103 bool hasValue() const {
1104 return K == Variable || K == VariableKnownToBeZero;
1105 }
1106
1107 SDValue getValue() const {
1108 assert(hasValue() && "Cannot get the value of a constant bit")((hasValue() && "Cannot get the value of a constant bit"
) ? static_cast<void> (0) : __assert_fail ("hasValue() && \"Cannot get the value of a constant bit\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1108, __PRETTY_FUNCTION__))
;
1109 return V;
1110 }
1111
1112 unsigned getValueBitIndex() const {
1113 assert(hasValue() && "Cannot get the value bit index of a constant bit")((hasValue() && "Cannot get the value bit index of a constant bit"
) ? static_cast<void> (0) : __assert_fail ("hasValue() && \"Cannot get the value bit index of a constant bit\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1113, __PRETTY_FUNCTION__))
;
1114 return Idx;
1115 }
1116 };
1117
1118 // A bit group has the same underlying value and the same rotate factor.
1119 struct BitGroup {
1120 SDValue V;
1121 unsigned RLAmt;
1122 unsigned StartIdx, EndIdx;
1123
1124 // This rotation amount assumes that the lower 32 bits of the quantity are
1125 // replicated in the high 32 bits by the rotation operator (which is done
1126 // by rlwinm and friends in 64-bit mode).
1127 bool Repl32;
1128 // Did converting to Repl32 == true change the rotation factor? If it did,
1129 // it decreased it by 32.
1130 bool Repl32CR;
1131 // Was this group coalesced after setting Repl32 to true?
1132 bool Repl32Coalesced;
1133
1134 BitGroup(SDValue V, unsigned R, unsigned S, unsigned E)
1135 : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false),
1136 Repl32Coalesced(false) {
1137 LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << Rdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tbit group for " <<
V.getNode() << " RLAmt = " << R << " [" <<
S << ", " << E << "]\n"; } } while (false)
1138 << " [" << S << ", " << E << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tbit group for " <<
V.getNode() << " RLAmt = " << R << " [" <<
S << ", " << E << "]\n"; } } while (false)
;
1139 }
1140 };
1141
1142 // Information on each (Value, RLAmt) pair (like the number of groups
1143 // associated with each) used to choose the lowering method.
1144 struct ValueRotInfo {
1145 SDValue V;
1146 unsigned RLAmt = std::numeric_limits<unsigned>::max();
1147 unsigned NumGroups = 0;
1148 unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max();
1149 bool Repl32 = false;
1150
1151 ValueRotInfo() = default;
1152
1153 // For sorting (in reverse order) by NumGroups, and then by
1154 // FirstGroupStartIdx.
1155 bool operator < (const ValueRotInfo &Other) const {
1156 // We need to sort so that the non-Repl32 come first because, when we're
1157 // doing masking, the Repl32 bit groups might be subsumed into the 64-bit
1158 // masking operation.
1159 if (Repl32 < Other.Repl32)
1160 return true;
1161 else if (Repl32 > Other.Repl32)
1162 return false;
1163 else if (NumGroups > Other.NumGroups)
1164 return true;
1165 else if (NumGroups < Other.NumGroups)
1166 return false;
1167 else if (RLAmt == 0 && Other.RLAmt != 0)
1168 return true;
1169 else if (RLAmt != 0 && Other.RLAmt == 0)
1170 return false;
1171 else if (FirstGroupStartIdx < Other.FirstGroupStartIdx)
1172 return true;
1173 return false;
1174 }
1175 };
1176
1177 using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>;
1178 using ValueBitsMemoizer =
1179 DenseMap<SDValue, std::unique_ptr<ValueBitsMemoizedValue>>;
1180 ValueBitsMemoizer Memoizer;
1181
1182 // Return a pair of bool and a SmallVector pointer to a memoization entry.
1183 // The bool is true if something interesting was deduced, otherwise if we're
1184 // providing only a generic representation of V (or something else likewise
1185 // uninteresting for instruction selection) through the SmallVector.
1186 std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V,
1187 unsigned NumBits) {
1188 auto &ValueEntry = Memoizer[V];
1189 if (ValueEntry)
1190 return std::make_pair(ValueEntry->first, &ValueEntry->second);
1191 ValueEntry.reset(new ValueBitsMemoizedValue());
1192 bool &Interesting = ValueEntry->first;
1193 SmallVector<ValueBit, 64> &Bits = ValueEntry->second;
1194 Bits.resize(NumBits);
1195
1196 switch (V.getOpcode()) {
1197 default: break;
1198 case ISD::ROTL:
1199 if (isa<ConstantSDNode>(V.getOperand(1))) {
1200 unsigned RotAmt = V.getConstantOperandVal(1);
1201
1202 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1203
1204 for (unsigned i = 0; i < NumBits; ++i)
1205 Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt];
1206
1207 return std::make_pair(Interesting = true, &Bits);
1208 }
1209 break;
1210 case ISD::SHL:
1211 if (isa<ConstantSDNode>(V.getOperand(1))) {
1212 unsigned ShiftAmt = V.getConstantOperandVal(1);
1213
1214 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1215
1216 for (unsigned i = ShiftAmt; i < NumBits; ++i)
1217 Bits[i] = LHSBits[i - ShiftAmt];
1218
1219 for (unsigned i = 0; i < ShiftAmt; ++i)
1220 Bits[i] = ValueBit(ValueBit::ConstZero);
1221
1222 return std::make_pair(Interesting = true, &Bits);
1223 }
1224 break;
1225 case ISD::SRL:
1226 if (isa<ConstantSDNode>(V.getOperand(1))) {
1227 unsigned ShiftAmt = V.getConstantOperandVal(1);
1228
1229 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1230
1231 for (unsigned i = 0; i < NumBits - ShiftAmt; ++i)
1232 Bits[i] = LHSBits[i + ShiftAmt];
1233
1234 for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i)
1235 Bits[i] = ValueBit(ValueBit::ConstZero);
1236
1237 return std::make_pair(Interesting = true, &Bits);
1238 }
1239 break;
1240 case ISD::AND:
1241 if (isa<ConstantSDNode>(V.getOperand(1))) {
1242 uint64_t Mask = V.getConstantOperandVal(1);
1243
1244 const SmallVector<ValueBit, 64> *LHSBits;
1245 // Mark this as interesting, only if the LHS was also interesting. This
1246 // prevents the overall procedure from matching a single immediate 'and'
1247 // (which is non-optimal because such an and might be folded with other
1248 // things if we don't select it here).
1249 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits);
1250
1251 for (unsigned i = 0; i < NumBits; ++i)
1252 if (((Mask >> i) & 1) == 1)
1253 Bits[i] = (*LHSBits)[i];
1254 else {
1255 // AND instruction masks this bit. If the input is already zero,
1256 // we have nothing to do here. Otherwise, make the bit ConstZero.
1257 if ((*LHSBits)[i].isZero())
1258 Bits[i] = (*LHSBits)[i];
1259 else
1260 Bits[i] = ValueBit(ValueBit::ConstZero);
1261 }
1262
1263 return std::make_pair(Interesting, &Bits);
1264 }
1265 break;
1266 case ISD::OR: {
1267 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1268 const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second;
1269
1270 bool AllDisjoint = true;
1271 SDValue LastVal = SDValue();
1272 unsigned LastIdx = 0;
1273 for (unsigned i = 0; i < NumBits; ++i) {
1274 if (LHSBits[i].isZero() && RHSBits[i].isZero()) {
1275 // If both inputs are known to be zero and one is ConstZero and
1276 // another is VariableKnownToBeZero, we can select whichever
1277 // we like. To minimize the number of bit groups, we select
1278 // VariableKnownToBeZero if this bit is the next bit of the same
1279 // input variable from the previous bit. Otherwise, we select
1280 // ConstZero.
1281 if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal &&
1282 LHSBits[i].getValueBitIndex() == LastIdx + 1)
1283 Bits[i] = LHSBits[i];
1284 else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal &&
1285 RHSBits[i].getValueBitIndex() == LastIdx + 1)
1286 Bits[i] = RHSBits[i];
1287 else
1288 Bits[i] = ValueBit(ValueBit::ConstZero);
1289 }
1290 else if (LHSBits[i].isZero())
1291 Bits[i] = RHSBits[i];
1292 else if (RHSBits[i].isZero())
1293 Bits[i] = LHSBits[i];
1294 else {
1295 AllDisjoint = false;
1296 break;
1297 }
1298 // We remember the value and bit index of this bit.
1299 if (Bits[i].hasValue()) {
1300 LastVal = Bits[i].getValue();
1301 LastIdx = Bits[i].getValueBitIndex();
1302 }
1303 else {
1304 if (LastVal) LastVal = SDValue();
1305 LastIdx = 0;
1306 }
1307 }
1308
1309 if (!AllDisjoint)
1310 break;
1311
1312 return std::make_pair(Interesting = true, &Bits);
1313 }
1314 case ISD::ZERO_EXTEND: {
1315 // We support only the case with zero extension from i32 to i64 so far.
1316 if (V.getValueType() != MVT::i64 ||
1317 V.getOperand(0).getValueType() != MVT::i32)
1318 break;
1319
1320 const SmallVector<ValueBit, 64> *LHSBits;
1321 const unsigned NumOperandBits = 32;
1322 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1323 NumOperandBits);
1324
1325 for (unsigned i = 0; i < NumOperandBits; ++i)
1326 Bits[i] = (*LHSBits)[i];
1327
1328 for (unsigned i = NumOperandBits; i < NumBits; ++i)
1329 Bits[i] = ValueBit(ValueBit::ConstZero);
1330
1331 return std::make_pair(Interesting, &Bits);
1332 }
1333 case ISD::TRUNCATE: {
1334 EVT FromType = V.getOperand(0).getValueType();
1335 EVT ToType = V.getValueType();
1336 // We support only the case with truncate from i64 to i32.
1337 if (FromType != MVT::i64 || ToType != MVT::i32)
1338 break;
1339 const unsigned NumAllBits = FromType.getSizeInBits();
1340 SmallVector<ValueBit, 64> *InBits;
1341 std::tie(Interesting, InBits) = getValueBits(V.getOperand(0),
1342 NumAllBits);
1343 const unsigned NumValidBits = ToType.getSizeInBits();
1344
1345 // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value.
1346 // So, we cannot include this truncate.
1347 bool UseUpper32bit = false;
1348 for (unsigned i = 0; i < NumValidBits; ++i)
1349 if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) {
1350 UseUpper32bit = true;
1351 break;
1352 }
1353 if (UseUpper32bit)
1354 break;
1355
1356 for (unsigned i = 0; i < NumValidBits; ++i)
1357 Bits[i] = (*InBits)[i];
1358
1359 return std::make_pair(Interesting, &Bits);
1360 }
1361 case ISD::AssertZext: {
1362 // For AssertZext, we look through the operand and
1363 // mark the bits known to be zero.
1364 const SmallVector<ValueBit, 64> *LHSBits;
1365 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1366 NumBits);
1367
1368 EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT();
1369 const unsigned NumValidBits = FromType.getSizeInBits();
1370 for (unsigned i = 0; i < NumValidBits; ++i)
1371 Bits[i] = (*LHSBits)[i];
1372
1373 // These bits are known to be zero.
1374 for (unsigned i = NumValidBits; i < NumBits; ++i)
1375 Bits[i] = ValueBit((*LHSBits)[i].getValue(),
1376 (*LHSBits)[i].getValueBitIndex(),
1377 ValueBit::VariableKnownToBeZero);
1378
1379 return std::make_pair(Interesting, &Bits);
1380 }
1381 case ISD::LOAD:
1382 LoadSDNode *LD = cast<LoadSDNode>(V);
1383 if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) {
1384 EVT VT = LD->getMemoryVT();
1385 const unsigned NumValidBits = VT.getSizeInBits();
1386
1387 for (unsigned i = 0; i < NumValidBits; ++i)
1388 Bits[i] = ValueBit(V, i);
1389
1390 // These bits are known to be zero.
1391 for (unsigned i = NumValidBits; i < NumBits; ++i)
1392 Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero);
1393
1394 // Zero-extending load itself cannot be optimized. So, it is not
1395 // interesting by itself though it gives useful information.
1396 return std::make_pair(Interesting = false, &Bits);
1397 }
1398 break;
1399 }
1400
1401 for (unsigned i = 0; i < NumBits; ++i)
1402 Bits[i] = ValueBit(V, i);
1403
1404 return std::make_pair(Interesting = false, &Bits);
1405 }
1406
1407 // For each value (except the constant ones), compute the left-rotate amount
1408 // to get it from its original to final position.
1409 void computeRotationAmounts() {
1410 NeedMask = false;
1411 RLAmt.resize(Bits.size());
1412 for (unsigned i = 0; i < Bits.size(); ++i)
1413 if (Bits[i].hasValue()) {
1414 unsigned VBI = Bits[i].getValueBitIndex();
1415 if (i >= VBI)
1416 RLAmt[i] = i - VBI;
1417 else
1418 RLAmt[i] = Bits.size() - (VBI - i);
1419 } else if (Bits[i].isZero()) {
1420 NeedMask = true;
1421 RLAmt[i] = UINT32_MAX(4294967295U);
1422 } else {
1423 llvm_unreachable("Unknown value bit type")::llvm::llvm_unreachable_internal("Unknown value bit type", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1423)
;
1424 }
1425 }
1426
1427 // Collect groups of consecutive bits with the same underlying value and
1428 // rotation factor. If we're doing late masking, we ignore zeros, otherwise
1429 // they break up groups.
1430 void collectBitGroups(bool LateMask) {
1431 BitGroups.clear();
1432
1433 unsigned LastRLAmt = RLAmt[0];
1434 SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue();
1435 unsigned LastGroupStartIdx = 0;
1436 bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1437 for (unsigned i = 1; i < Bits.size(); ++i) {
1438 unsigned ThisRLAmt = RLAmt[i];
1439 SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue();
1440 if (LateMask && !ThisValue) {
1441 ThisValue = LastValue;
1442 ThisRLAmt = LastRLAmt;
1443 // If we're doing late masking, then the first bit group always starts
1444 // at zero (even if the first bits were zero).
1445 if (BitGroups.empty())
1446 LastGroupStartIdx = 0;
1447 }
1448
1449 // If this bit is known to be zero and the current group is a bit group
1450 // of zeros, we do not need to terminate the current bit group even the
1451 // Value or RLAmt does not match here. Instead, we terminate this group
1452 // when the first non-zero bit appears later.
1453 if (IsGroupOfZeros && Bits[i].isZero())
1454 continue;
1455
1456 // If this bit has the same underlying value and the same rotate factor as
1457 // the last one, then they're part of the same group.
1458 if (ThisRLAmt == LastRLAmt && ThisValue == LastValue)
1459 // We cannot continue the current group if this bits is not known to
1460 // be zero in a bit group of zeros.
1461 if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero()))
1462 continue;
1463
1464 if (LastValue.getNode())
1465 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1466 i-1));
1467 LastRLAmt = ThisRLAmt;
1468 LastValue = ThisValue;
1469 LastGroupStartIdx = i;
1470 IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1471 }
1472 if (LastValue.getNode())
1473 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1474 Bits.size()-1));
1475
1476 if (BitGroups.empty())
1477 return;
1478
1479 // We might be able to combine the first and last groups.
1480 if (BitGroups.size() > 1) {
1481 // If the first and last groups are the same, then remove the first group
1482 // in favor of the last group, making the ending index of the last group
1483 // equal to the ending index of the to-be-removed first group.
1484 if (BitGroups[0].StartIdx == 0 &&
1485 BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 &&
1486 BitGroups[0].V == BitGroups[BitGroups.size()-1].V &&
1487 BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) {
1488 LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining final bit group with initial one\n"
; } } while (false)
;
1489 BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx;
1490 BitGroups.erase(BitGroups.begin());
1491 }
1492 }
1493 }
1494
1495 // Take all (SDValue, RLAmt) pairs and sort them by the number of groups
1496 // associated with each. If the number of groups are same, we prefer a group
1497 // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate
1498 // instruction. If there is a degeneracy, pick the one that occurs
1499 // first (in the final value).
1500 void collectValueRotInfo() {
1501 ValueRots.clear();
1502
1503 for (auto &BG : BitGroups) {
1504 unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0);
1505 ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)];
1506 VRI.V = BG.V;
1507 VRI.RLAmt = BG.RLAmt;
1508 VRI.Repl32 = BG.Repl32;
1509 VRI.NumGroups += 1;
1510 VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx);
1511 }
1512
1513 // Now that we've collected the various ValueRotInfo instances, we need to
1514 // sort them.
1515 ValueRotsVec.clear();
1516 for (auto &I : ValueRots) {
1517 ValueRotsVec.push_back(I.second);
1518 }
1519 llvm::sort(ValueRotsVec);
1520 }
1521
1522 // In 64-bit mode, rlwinm and friends have a rotation operator that
1523 // replicates the low-order 32 bits into the high-order 32-bits. The mask
1524 // indices of these instructions can only be in the lower 32 bits, so they
1525 // can only represent some 64-bit bit groups. However, when they can be used,
1526 // the 32-bit replication can be used to represent, as a single bit group,
1527 // otherwise separate bit groups. We'll convert to replicated-32-bit bit
1528 // groups when possible. Returns true if any of the bit groups were
1529 // converted.
1530 void assignRepl32BitGroups() {
1531 // If we have bits like this:
1532 //
1533 // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
1534 // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24
1535 // Groups: | RLAmt = 8 | RLAmt = 40 |
1536 //
1537 // But, making use of a 32-bit operation that replicates the low-order 32
1538 // bits into the high-order 32 bits, this can be one bit group with a RLAmt
1539 // of 8.
1540
1541 auto IsAllLow32 = [this](BitGroup & BG) {
1542 if (BG.StartIdx <= BG.EndIdx) {
1543 for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) {
1544 if (!Bits[i].hasValue())
1545 continue;
1546 if (Bits[i].getValueBitIndex() >= 32)
1547 return false;
1548 }
1549 } else {
1550 for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) {
1551 if (!Bits[i].hasValue())
1552 continue;
1553 if (Bits[i].getValueBitIndex() >= 32)
1554 return false;
1555 }
1556 for (unsigned i = 0; i <= BG.EndIdx; ++i) {
1557 if (!Bits[i].hasValue())
1558 continue;
1559 if (Bits[i].getValueBitIndex() >= 32)
1560 return false;
1561 }
1562 }
1563
1564 return true;
1565 };
1566
1567 for (auto &BG : BitGroups) {
1568 // If this bit group has RLAmt of 0 and will not be merged with
1569 // another bit group, we don't benefit from Repl32. We don't mark
1570 // such group to give more freedom for later instruction selection.
1571 if (BG.RLAmt == 0) {
1572 auto PotentiallyMerged = [this](BitGroup & BG) {
1573 for (auto &BG2 : BitGroups)
1574 if (&BG != &BG2 && BG.V == BG2.V &&
1575 (BG2.RLAmt == 0 || BG2.RLAmt == 32))
1576 return true;
1577 return false;
1578 };
1579 if (!PotentiallyMerged(BG))
1580 continue;
1581 }
1582 if (BG.StartIdx < 32 && BG.EndIdx < 32) {
1583 if (IsAllLow32(BG)) {
1584 if (BG.RLAmt >= 32) {
1585 BG.RLAmt -= 32;
1586 BG.Repl32CR = true;
1587 }
1588
1589 BG.Repl32 = true;
1590
1591 LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t32-bit replicated bit group for "
<< BG.V.getNode() << " RLAmt = " << BG.RLAmt
<< " [" << BG.StartIdx << ", " << BG
.EndIdx << "]\n"; } } while (false)
1592 << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " ["do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t32-bit replicated bit group for "
<< BG.V.getNode() << " RLAmt = " << BG.RLAmt
<< " [" << BG.StartIdx << ", " << BG
.EndIdx << "]\n"; } } while (false)
1593 << BG.StartIdx << ", " << BG.EndIdx << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t32-bit replicated bit group for "
<< BG.V.getNode() << " RLAmt = " << BG.RLAmt
<< " [" << BG.StartIdx << ", " << BG
.EndIdx << "]\n"; } } while (false)
;
1594 }
1595 }
1596 }
1597
1598 // Now walk through the bit groups, consolidating where possible.
1599 for (auto I = BitGroups.begin(); I != BitGroups.end();) {
1600 // We might want to remove this bit group by merging it with the previous
1601 // group (which might be the ending group).
1602 auto IP = (I == BitGroups.begin()) ?
1603 std::prev(BitGroups.end()) : std::prev(I);
1604 if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt &&
1605 I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) {
1606
1607 LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with group with range [" << IP
->StartIdx << ", " << IP->EndIdx << "]\n"
; } } while (false)
1608 << I->V.getNode() << " RLAmt = " << I->RLAmt << " ["do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with group with range [" << IP
->StartIdx << ", " << IP->EndIdx << "]\n"
; } } while (false)
1609 << I->StartIdx << ", " << I->EndIdxdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with group with range [" << IP
->StartIdx << ", " << IP->EndIdx << "]\n"
; } } while (false)
1610 << "] with group with range [" << IP->StartIdx << ", "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with group with range [" << IP
->StartIdx << ", " << IP->EndIdx << "]\n"
; } } while (false)
1611 << IP->EndIdx << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with group with range [" << IP
->StartIdx << ", " << IP->EndIdx << "]\n"
; } } while (false)
;
1612
1613 IP->EndIdx = I->EndIdx;
1614 IP->Repl32CR = IP->Repl32CR || I->Repl32CR;
1615 IP->Repl32Coalesced = true;
1616 I = BitGroups.erase(I);
1617 continue;
1618 } else {
1619 // There is a special case worth handling: If there is a single group
1620 // covering the entire upper 32 bits, and it can be merged with both
1621 // the next and previous groups (which might be the same group), then
1622 // do so. If it is the same group (so there will be only one group in
1623 // total), then we need to reverse the order of the range so that it
1624 // covers the entire 64 bits.
1625 if (I->StartIdx == 32 && I->EndIdx == 63) {
1626 assert(std::next(I) == BitGroups.end() &&((std::next(I) == BitGroups.end() && "bit group ends at index 63 but there is another?"
) ? static_cast<void> (0) : __assert_fail ("std::next(I) == BitGroups.end() && \"bit group ends at index 63 but there is another?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1627, __PRETTY_FUNCTION__))
1627 "bit group ends at index 63 but there is another?")((std::next(I) == BitGroups.end() && "bit group ends at index 63 but there is another?"
) ? static_cast<void> (0) : __assert_fail ("std::next(I) == BitGroups.end() && \"bit group ends at index 63 but there is another?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1627, __PRETTY_FUNCTION__))
;
1628 auto IN = BitGroups.begin();
1629
1630 if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V &&
1631 (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt &&
1632 IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP &&
1633 IsAllLow32(*I)) {
1634
1635 LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with 32-bit replicated groups with ranges ["
<< IP->StartIdx << ", " << IP->EndIdx
<< "] and [" << IN->StartIdx << ", " <<
IN->EndIdx << "]\n"; } } while (false)
1636 << " RLAmt = " << I->RLAmt << " [" << I->StartIdxdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with 32-bit replicated groups with ranges ["
<< IP->StartIdx << ", " << IP->EndIdx
<< "] and [" << IN->StartIdx << ", " <<
IN->EndIdx << "]\n"; } } while (false)
1637 << ", " << I->EndIdxdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with 32-bit replicated groups with ranges ["
<< IP->StartIdx << ", " << IP->EndIdx
<< "] and [" << IN->StartIdx << ", " <<
IN->EndIdx << "]\n"; } } while (false)
1638 << "] with 32-bit replicated groups with ranges ["do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with 32-bit replicated groups with ranges ["
<< IP->StartIdx << ", " << IP->EndIdx
<< "] and [" << IN->StartIdx << ", " <<
IN->EndIdx << "]\n"; } } while (false)
1639 << IP->StartIdx << ", " << IP->EndIdx << "] and ["do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with 32-bit replicated groups with ranges ["
<< IP->StartIdx << ", " << IP->EndIdx
<< "] and [" << IN->StartIdx << ", " <<
IN->EndIdx << "]\n"; } } while (false)
1640 << IN->StartIdx << ", " << IN->EndIdx << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tcombining bit group for "
<< I->V.getNode() << " RLAmt = " << I->
RLAmt << " [" << I->StartIdx << ", " <<
I->EndIdx << "] with 32-bit replicated groups with ranges ["
<< IP->StartIdx << ", " << IP->EndIdx
<< "] and [" << IN->StartIdx << ", " <<
IN->EndIdx << "]\n"; } } while (false)
;
1641
1642 if (IP == IN) {
1643 // There is only one other group; change it to cover the whole
1644 // range (backward, so that it can still be Repl32 but cover the
1645 // whole 64-bit range).
1646 IP->StartIdx = 31;
1647 IP->EndIdx = 30;
1648 IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32;
1649 IP->Repl32Coalesced = true;
1650 I = BitGroups.erase(I);
1651 } else {
1652 // There are two separate groups, one before this group and one
1653 // after us (at the beginning). We're going to remove this group,
1654 // but also the group at the very beginning.
1655 IP->EndIdx = IN->EndIdx;
1656 IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32;
1657 IP->Repl32Coalesced = true;
1658 I = BitGroups.erase(I);
1659 BitGroups.erase(BitGroups.begin());
1660 }
1661
1662 // This must be the last group in the vector (and we might have
1663 // just invalidated the iterator above), so break here.
1664 break;
1665 }
1666 }
1667 }
1668
1669 ++I;
1670 }
1671 }
1672
1673 SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
1674 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1675 }
1676
1677 uint64_t getZerosMask() {
1678 uint64_t Mask = 0;
1679 for (unsigned i = 0; i < Bits.size(); ++i) {
1680 if (Bits[i].hasValue())
1681 continue;
1682 Mask |= (UINT64_C(1)1UL << i);
1683 }
1684
1685 return ~Mask;
1686 }
1687
1688 // This method extends an input value to 64 bit if input is 32-bit integer.
1689 // While selecting instructions in BitPermutationSelector in 64-bit mode,
1690 // an input value can be a 32-bit integer if a ZERO_EXTEND node is included.
1691 // In such case, we extend it to 64 bit to be consistent with other values.
1692 SDValue ExtendToInt64(SDValue V, const SDLoc &dl) {
1693 if (V.getValueSizeInBits() == 64)
1694 return V;
1695
1696 assert(V.getValueSizeInBits() == 32)((V.getValueSizeInBits() == 32) ? static_cast<void> (0)
: __assert_fail ("V.getValueSizeInBits() == 32", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1696, __PRETTY_FUNCTION__))
;
1697 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1698 SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
1699 MVT::i64), 0);
1700 SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
1701 MVT::i64, ImDef, V,
1702 SubRegIdx), 0);
1703 return ExtVal;
1704 }
1705
1706 SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
1707 if (V.getValueSizeInBits() == 32)
1708 return V;
1709
1710 assert(V.getValueSizeInBits() == 64)((V.getValueSizeInBits() == 64) ? static_cast<void> (0)
: __assert_fail ("V.getValueSizeInBits() == 64", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1710, __PRETTY_FUNCTION__))
;
1711 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1712 SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
1713 MVT::i32, V, SubRegIdx), 0);
1714 return SubVal;
1715 }
1716
1717 // Depending on the number of groups for a particular value, it might be
1718 // better to rotate, mask explicitly (using andi/andis), and then or the
1719 // result. Select this part of the result first.
1720 void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
1721 if (BPermRewriterNoMasking)
1722 return;
1723
1724 for (ValueRotInfo &VRI : ValueRotsVec) {
1725 unsigned Mask = 0;
1726 for (unsigned i = 0; i < Bits.size(); ++i) {
1727 if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V)
1728 continue;
1729 if (RLAmt[i] != VRI.RLAmt)
1730 continue;
1731 Mask |= (1u << i);
1732 }
1733
1734 // Compute the masks for andi/andis that would be necessary.
1735 unsigned ANDIMask = (Mask & UINT16_MAX(65535)), ANDISMask = Mask >> 16;
1736 assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask for value bit groups"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask for value bit groups\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1737, __PRETTY_FUNCTION__))
1737 "No set bits in mask for value bit groups")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask for value bit groups"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask for value bit groups\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1737, __PRETTY_FUNCTION__))
;
1738 bool NeedsRotate = VRI.RLAmt != 0;
1739
1740 // We're trying to minimize the number of instructions. If we have one
1741 // group, using one of andi/andis can break even. If we have three
1742 // groups, we can use both andi and andis and break even (to use both
1743 // andi and andis we also need to or the results together). We need four
1744 // groups if we also need to rotate. To use andi/andis we need to do more
1745 // than break even because rotate-and-mask instructions tend to be easier
1746 // to schedule.
1747
1748 // FIXME: We've biased here against using andi/andis, which is right for
1749 // POWER cores, but not optimal everywhere. For example, on the A2,
1750 // andi/andis have single-cycle latency whereas the rotate-and-mask
1751 // instructions take two cycles, and it would be better to bias toward
1752 // andi/andis in break-even cases.
1753
1754 unsigned NumAndInsts = (unsigned) NeedsRotate +
1755 (unsigned) (ANDIMask != 0) +
1756 (unsigned) (ANDISMask != 0) +
1757 (unsigned) (ANDIMask != 0 && ANDISMask != 0) +
1758 (unsigned) (bool) Res;
1759
1760 LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< ":" << "\n\t\t\tisel using masking: " <<
NumAndInsts << " using rotates: " << VRI.NumGroups
<< "\n"; } } while (false)
1761 << " RL: " << VRI.RLAmt << ":"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< ":" << "\n\t\t\tisel using masking: " <<
NumAndInsts << " using rotates: " << VRI.NumGroups
<< "\n"; } } while (false)
1762 << "\n\t\t\tisel using masking: " << NumAndInstsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< ":" << "\n\t\t\tisel using masking: " <<
NumAndInsts << " using rotates: " << VRI.NumGroups
<< "\n"; } } while (false)
1763 << " using rotates: " << VRI.NumGroups << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< ":" << "\n\t\t\tisel using masking: " <<
NumAndInsts << " using rotates: " << VRI.NumGroups
<< "\n"; } } while (false)
;
1764
1765 if (NumAndInsts >= VRI.NumGroups)
1766 continue;
1767
1768 LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\t\t\tusing masking\n";
} } while (false)
;
1769
1770 if (InstCnt) *InstCnt += NumAndInsts;
1771
1772 SDValue VRot;
1773 if (VRI.RLAmt) {
1774 SDValue Ops[] =
1775 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
1776 getI32Imm(0, dl), getI32Imm(31, dl) };
1777 VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
1778 Ops), 0);
1779 } else {
1780 VRot = TruncateToInt32(VRI.V, dl);
1781 }
1782
1783 SDValue ANDIVal, ANDISVal;
1784 if (ANDIMask != 0)
1785 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo, dl, MVT::i32,
1786 VRot, getI32Imm(ANDIMask, dl)), 0);
1787 if (ANDISMask != 0)
1788 ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo, dl, MVT::i32,
1789 VRot, getI32Imm(ANDISMask, dl)), 0);
1790
1791 SDValue TotalVal;
1792 if (!ANDIVal)
1793 TotalVal = ANDISVal;
1794 else if (!ANDISVal)
1795 TotalVal = ANDIVal;
1796 else
1797 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
1798 ANDIVal, ANDISVal), 0);
1799
1800 if (!Res)
1801 Res = TotalVal;
1802 else
1803 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
1804 Res, TotalVal), 0);
1805
1806 // Now, remove all groups with this underlying value and rotation
1807 // factor.
1808 eraseMatchingBitGroups([VRI](const BitGroup &BG) {
1809 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
1810 });
1811 }
1812 }
1813
1814 // Instruction selection for the 32-bit case.
1815 SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) {
1816 SDLoc dl(N);
1817 SDValue Res;
1818
1819 if (InstCnt) *InstCnt = 0;
1820
1821 // Take care of cases that should use andi/andis first.
1822 SelectAndParts32(dl, Res, InstCnt);
1823
1824 // If we've not yet selected a 'starting' instruction, and we have no zeros
1825 // to fill in, select the (Value, RLAmt) with the highest priority (largest
1826 // number of groups), and start with this rotated value.
1827 if ((!NeedMask || LateMask) && !Res) {
1828 ValueRotInfo &VRI = ValueRotsVec[0];
1829 if (VRI.RLAmt) {
1830 if (InstCnt) *InstCnt += 1;
1831 SDValue Ops[] =
1832 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
1833 getI32Imm(0, dl), getI32Imm(31, dl) };
1834 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
1835 0);
1836 } else {
1837 Res = TruncateToInt32(VRI.V, dl);
1838 }
1839
1840 // Now, remove all groups with this underlying value and rotation factor.
1841 eraseMatchingBitGroups([VRI](const BitGroup &BG) {
1842 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
1843 });
1844 }
1845
1846 if (InstCnt) *InstCnt += BitGroups.size();
1847
1848 // Insert the other groups (one at a time).
1849 for (auto &BG : BitGroups) {
1850 if (!Res) {
1851 SDValue Ops[] =
1852 { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
1853 getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
1854 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
1855 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
1856 } else {
1857 SDValue Ops[] =
1858 { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
1859 getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
1860 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
1861 Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0);
1862 }
1863 }
1864
1865 if (LateMask) {
1866 unsigned Mask = (unsigned) getZerosMask();
1867
1868 unsigned ANDIMask = (Mask & UINT16_MAX(65535)), ANDISMask = Mask >> 16;
1869 assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in zeros mask?"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in zeros mask?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1870, __PRETTY_FUNCTION__))
1870 "No set bits in zeros mask?")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in zeros mask?"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in zeros mask?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1870, __PRETTY_FUNCTION__))
;
1871
1872 if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
1873 (unsigned) (ANDISMask != 0) +
1874 (unsigned) (ANDIMask != 0 && ANDISMask != 0);
1875
1876 SDValue ANDIVal, ANDISVal;
1877 if (ANDIMask != 0)
1878 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo, dl, MVT::i32,
1879 Res, getI32Imm(ANDIMask, dl)), 0);
1880 if (ANDISMask != 0)
1881 ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo, dl, MVT::i32,
1882 Res, getI32Imm(ANDISMask, dl)), 0);
1883
1884 if (!ANDIVal)
1885 Res = ANDISVal;
1886 else if (!ANDISVal)
1887 Res = ANDIVal;
1888 else
1889 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
1890 ANDIVal, ANDISVal), 0);
1891 }
1892
1893 return Res.getNode();
1894 }
1895
1896 unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32,
1897 unsigned MaskStart, unsigned MaskEnd,
1898 bool IsIns) {
1899 // In the notation used by the instructions, 'start' and 'end' are reversed
1900 // because bits are counted from high to low order.
1901 unsigned InstMaskStart = 64 - MaskEnd - 1,
1902 InstMaskEnd = 64 - MaskStart - 1;
1903
1904 if (Repl32)
1905 return 1;
1906
1907 if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) ||
1908 InstMaskEnd == 63 - RLAmt)
1909 return 1;
1910
1911 return 2;
1912 }
1913
1914 // For 64-bit values, not all combinations of rotates and masks are
1915 // available. Produce one if it is available.
1916 SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt,
1917 bool Repl32, unsigned MaskStart, unsigned MaskEnd,
1918 unsigned *InstCnt = nullptr) {
1919 // In the notation used by the instructions, 'start' and 'end' are reversed
1920 // because bits are counted from high to low order.
1921 unsigned InstMaskStart = 64 - MaskEnd - 1,
1922 InstMaskEnd = 64 - MaskStart - 1;
1923
1924 if (InstCnt) *InstCnt += 1;
1925
1926 if (Repl32) {
1927 // This rotation amount assumes that the lower 32 bits of the quantity
1928 // are replicated in the high 32 bits by the rotation operator (which is
1929 // done by rlwinm and friends).
1930 assert(InstMaskStart >= 32 && "Mask cannot start out of range")((InstMaskStart >= 32 && "Mask cannot start out of range"
) ? static_cast<void> (0) : __assert_fail ("InstMaskStart >= 32 && \"Mask cannot start out of range\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1930, __PRETTY_FUNCTION__))
;
1931 assert(InstMaskEnd >= 32 && "Mask cannot end out of range")((InstMaskEnd >= 32 && "Mask cannot end out of range"
) ? static_cast<void> (0) : __assert_fail ("InstMaskEnd >= 32 && \"Mask cannot end out of range\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1931, __PRETTY_FUNCTION__))
;
1932 SDValue Ops[] =
1933 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1934 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
1935 return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64,
1936 Ops), 0);
1937 }
1938
1939 if (InstMaskEnd == 63) {
1940 SDValue Ops[] =
1941 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1942 getI32Imm(InstMaskStart, dl) };
1943 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0);
1944 }
1945
1946 if (InstMaskStart == 0) {
1947 SDValue Ops[] =
1948 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1949 getI32Imm(InstMaskEnd, dl) };
1950 return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0);
1951 }
1952
1953 if (InstMaskEnd == 63 - RLAmt) {
1954 SDValue Ops[] =
1955 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1956 getI32Imm(InstMaskStart, dl) };
1957 return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0);
1958 }
1959
1960 // We cannot do this with a single instruction, so we'll use two. The
1961 // problem is that we're not free to choose both a rotation amount and mask
1962 // start and end independently. We can choose an arbitrary mask start and
1963 // end, but then the rotation amount is fixed. Rotation, however, can be
1964 // inverted, and so by applying an "inverse" rotation first, we can get the
1965 // desired result.
1966 if (InstCnt) *InstCnt += 1;
1967
1968 // The rotation mask for the second instruction must be MaskStart.
1969 unsigned RLAmt2 = MaskStart;
1970 // The first instruction must rotate V so that the overall rotation amount
1971 // is RLAmt.
1972 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
1973 if (RLAmt1)
1974 V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
1975 return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd);
1976 }
1977
1978 // For 64-bit values, not all combinations of rotates and masks are
1979 // available. Produce a rotate-mask-and-insert if one is available.
1980 SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
1981 unsigned RLAmt, bool Repl32, unsigned MaskStart,
1982 unsigned MaskEnd, unsigned *InstCnt = nullptr) {
1983 // In the notation used by the instructions, 'start' and 'end' are reversed
1984 // because bits are counted from high to low order.
1985 unsigned InstMaskStart = 64 - MaskEnd - 1,
1986 InstMaskEnd = 64 - MaskStart - 1;
1987
1988 if (InstCnt) *InstCnt += 1;
1989
1990 if (Repl32) {
1991 // This rotation amount assumes that the lower 32 bits of the quantity
1992 // are replicated in the high 32 bits by the rotation operator (which is
1993 // done by rlwinm and friends).
1994 assert(InstMaskStart >= 32 && "Mask cannot start out of range")((InstMaskStart >= 32 && "Mask cannot start out of range"
) ? static_cast<void> (0) : __assert_fail ("InstMaskStart >= 32 && \"Mask cannot start out of range\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1994, __PRETTY_FUNCTION__))
;
1995 assert(InstMaskEnd >= 32 && "Mask cannot end out of range")((InstMaskEnd >= 32 && "Mask cannot end out of range"
) ? static_cast<void> (0) : __assert_fail ("InstMaskEnd >= 32 && \"Mask cannot end out of range\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 1995, __PRETTY_FUNCTION__))
;
1996 SDValue Ops[] =
1997 { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1998 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
1999 return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64,
2000 Ops), 0);
2001 }
2002
2003 if (InstMaskEnd == 63 - RLAmt) {
2004 SDValue Ops[] =
2005 { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2006 getI32Imm(InstMaskStart, dl) };
2007 return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0);
2008 }
2009
2010 // We cannot do this with a single instruction, so we'll use two. The
2011 // problem is that we're not free to choose both a rotation amount and mask
2012 // start and end independently. We can choose an arbitrary mask start and
2013 // end, but then the rotation amount is fixed. Rotation, however, can be
2014 // inverted, and so by applying an "inverse" rotation first, we can get the
2015 // desired result.
2016 if (InstCnt) *InstCnt += 1;
2017
2018 // The rotation mask for the second instruction must be MaskStart.
2019 unsigned RLAmt2 = MaskStart;
2020 // The first instruction must rotate V so that the overall rotation amount
2021 // is RLAmt.
2022 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2023 if (RLAmt1)
2024 V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2025 return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd);
2026 }
2027
2028 void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2029 if (BPermRewriterNoMasking)
2030 return;
2031
2032 // The idea here is the same as in the 32-bit version, but with additional
2033 // complications from the fact that Repl32 might be true. Because we
2034 // aggressively convert bit groups to Repl32 form (which, for small
2035 // rotation factors, involves no other change), and then coalesce, it might
2036 // be the case that a single 64-bit masking operation could handle both
2037 // some Repl32 groups and some non-Repl32 groups. If converting to Repl32
2038 // form allowed coalescing, then we must use a 32-bit rotaton in order to
2039 // completely capture the new combined bit group.
2040
2041 for (ValueRotInfo &VRI : ValueRotsVec) {
2042 uint64_t Mask = 0;
2043
2044 // We need to add to the mask all bits from the associated bit groups.
2045 // If Repl32 is false, we need to add bits from bit groups that have
2046 // Repl32 true, but are trivially convertable to Repl32 false. Such a
2047 // group is trivially convertable if it overlaps only with the lower 32
2048 // bits, and the group has not been coalesced.
2049 auto MatchingBG = [VRI](const BitGroup &BG) {
2050 if (VRI.V != BG.V)
2051 return false;
2052
2053 unsigned EffRLAmt = BG.RLAmt;
2054 if (!VRI.Repl32 && BG.Repl32) {
2055 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx &&
2056 !BG.Repl32Coalesced) {
2057 if (BG.Repl32CR)
2058 EffRLAmt += 32;
2059 } else {
2060 return false;
2061 }
2062 } else if (VRI.Repl32 != BG.Repl32) {
2063 return false;
2064 }
2065
2066 return VRI.RLAmt == EffRLAmt;
2067 };
2068
2069 for (auto &BG : BitGroups) {
2070 if (!MatchingBG(BG))
2071 continue;
2072
2073 if (BG.StartIdx <= BG.EndIdx) {
2074 for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i)
2075 Mask |= (UINT64_C(1)1UL << i);
2076 } else {
2077 for (unsigned i = BG.StartIdx; i < Bits.size(); ++i)
2078 Mask |= (UINT64_C(1)1UL << i);
2079 for (unsigned i = 0; i <= BG.EndIdx; ++i)
2080 Mask |= (UINT64_C(1)1UL << i);
2081 }
2082 }
2083
2084 // We can use the 32-bit andi/andis technique if the mask does not
2085 // require any higher-order bits. This can save an instruction compared
2086 // to always using the general 64-bit technique.
2087 bool Use32BitInsts = isUInt<32>(Mask);
2088 // Compute the masks for andi/andis that would be necessary.
2089 unsigned ANDIMask = (Mask & UINT16_MAX(65535)),
2090 ANDISMask = (Mask >> 16) & UINT16_MAX(65535);
2091
2092 bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask));
2093
2094 unsigned NumAndInsts = (unsigned) NeedsRotate +
2095 (unsigned) (bool) Res;
2096 if (Use32BitInsts)
2097 NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) +
2098 (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2099 else
2100 NumAndInsts += selectI64ImmInstrCount(Mask) + /* and */ 1;
2101
2102 unsigned NumRLInsts = 0;
2103 bool FirstBG = true;
2104 bool MoreBG = false;
2105 for (auto &BG : BitGroups) {
2106 if (!MatchingBG(BG)) {
2107 MoreBG = true;
2108 continue;
2109 }
2110 NumRLInsts +=
2111 SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx,
2112 !FirstBG);
2113 FirstBG = false;
2114 }
2115
2116 LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: "
<< NumAndInsts << " using rotates: " << NumRLInsts
<< "\n"; } } while (false)
2117 << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: "
<< NumAndInsts << " using rotates: " << NumRLInsts
<< "\n"; } } while (false)
2118 << "\n\t\t\tisel using masking: " << NumAndInstsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: "
<< NumAndInsts << " using rotates: " << NumRLInsts
<< "\n"; } } while (false)
2119 << " using rotates: " << NumRLInsts << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\trotation groups for "
<< VRI.V.getNode() << " RL: " << VRI.RLAmt
<< (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: "
<< NumAndInsts << " using rotates: " << NumRLInsts
<< "\n"; } } while (false)
;
2120
2121 // When we'd use andi/andis, we bias toward using the rotates (andi only
2122 // has a record form, and is cracked on POWER cores). However, when using
2123 // general 64-bit constant formation, bias toward the constant form,
2124 // because that exposes more opportunities for CSE.
2125 if (NumAndInsts > NumRLInsts)
2126 continue;
2127 // When merging multiple bit groups, instruction or is used.
2128 // But when rotate is used, rldimi can inert the rotated value into any
2129 // register, so instruction or can be avoided.
2130 if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts)
2131 continue;
2132
2133 LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\t\t\tusing masking\n";
} } while (false)
;
2134
2135 if (InstCnt) *InstCnt += NumAndInsts;
2136
2137 SDValue VRot;
2138 // We actually need to generate a rotation if we have a non-zero rotation
2139 // factor or, in the Repl32 case, if we care about any of the
2140 // higher-order replicated bits. In the latter case, we generate a mask
2141 // backward so that it actually includes the entire 64 bits.
2142 if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)))
2143 VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2144 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63);
2145 else
2146 VRot = VRI.V;
2147
2148 SDValue TotalVal;
2149 if (Use32BitInsts) {
2150 assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2151, __PRETTY_FUNCTION__))
2151 "No set bits in mask when using 32-bit ands for 64-bit value")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2151, __PRETTY_FUNCTION__))
;
2152
2153 SDValue ANDIVal, ANDISVal;
2154 if (ANDIMask != 0)
2155 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo8, dl, MVT::i64,
2156 ExtendToInt64(VRot, dl),
2157 getI32Imm(ANDIMask, dl)),
2158 0);
2159 if (ANDISMask != 0)
2160 ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo8, dl, MVT::i64,
2161 ExtendToInt64(VRot, dl),
2162 getI32Imm(ANDISMask, dl)),
2163 0);
2164
2165 if (!ANDIVal)
2166 TotalVal = ANDISVal;
2167 else if (!ANDISVal)
2168 TotalVal = ANDIVal;
2169 else
2170 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2171 ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2172 } else {
2173 TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0);
2174 TotalVal =
2175 SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2176 ExtendToInt64(VRot, dl), TotalVal),
2177 0);
2178 }
2179
2180 if (!Res)
2181 Res = TotalVal;
2182 else
2183 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2184 ExtendToInt64(Res, dl), TotalVal),
2185 0);
2186
2187 // Now, remove all groups with this underlying value and rotation
2188 // factor.
2189 eraseMatchingBitGroups(MatchingBG);
2190 }
2191 }
2192
2193 // Instruction selection for the 64-bit case.
2194 SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) {
2195 SDLoc dl(N);
2196 SDValue Res;
2197
2198 if (InstCnt) *InstCnt = 0;
2199
2200 // Take care of cases that should use andi/andis first.
2201 SelectAndParts64(dl, Res, InstCnt);
2202
2203 // If we've not yet selected a 'starting' instruction, and we have no zeros
2204 // to fill in, select the (Value, RLAmt) with the highest priority (largest
2205 // number of groups), and start with this rotated value.
2206 if ((!NeedMask || LateMask) && !Res) {
2207 // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32
2208 // groups will come first, and so the VRI representing the largest number
2209 // of groups might not be first (it might be the first Repl32 groups).
2210 unsigned MaxGroupsIdx = 0;
2211 if (!ValueRotsVec[0].Repl32) {
2212 for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i)
2213 if (ValueRotsVec[i].Repl32) {
2214 if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups)
2215 MaxGroupsIdx = i;
2216 break;
2217 }
2218 }
2219
2220 ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx];
2221 bool NeedsRotate = false;
2222 if (VRI.RLAmt) {
2223 NeedsRotate = true;
2224 } else if (VRI.Repl32) {
2225 for (auto &BG : BitGroups) {
2226 if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt ||
2227 BG.Repl32 != VRI.Repl32)
2228 continue;
2229
2230 // We don't need a rotate if the bit group is confined to the lower
2231 // 32 bits.
2232 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx)
2233 continue;
2234
2235 NeedsRotate = true;
2236 break;
2237 }
2238 }
2239
2240 if (NeedsRotate)
2241 Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2242 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63,
2243 InstCnt);
2244 else
2245 Res = VRI.V;
2246
2247 // Now, remove all groups with this underlying value and rotation factor.
2248 if (Res)
2249 eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2250 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt &&
2251 BG.Repl32 == VRI.Repl32;
2252 });
2253 }
2254
2255 // Because 64-bit rotates are more flexible than inserts, we might have a
2256 // preference regarding which one we do first (to save one instruction).
2257 if (!Res)
2258 for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) {
2259 if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2260 false) <
2261 SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2262 true)) {
2263 if (I != BitGroups.begin()) {
2264 BitGroup BG = *I;
2265 BitGroups.erase(I);
2266 BitGroups.insert(BitGroups.begin(), BG);
2267 }
2268
2269 break;
2270 }
2271 }
2272
2273 // Insert the other groups (one at a time).
2274 for (auto &BG : BitGroups) {
2275 if (!Res)
2276 Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx,
2277 BG.EndIdx, InstCnt);
2278 else
2279 Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32,
2280 BG.StartIdx, BG.EndIdx, InstCnt);
2281 }
2282
2283 if (LateMask) {
2284 uint64_t Mask = getZerosMask();
2285
2286 // We can use the 32-bit andi/andis technique if the mask does not
2287 // require any higher-order bits. This can save an instruction compared
2288 // to always using the general 64-bit technique.
2289 bool Use32BitInsts = isUInt<32>(Mask);
2290 // Compute the masks for andi/andis that would be necessary.
2291 unsigned ANDIMask = (Mask & UINT16_MAX(65535)),
2292 ANDISMask = (Mask >> 16) & UINT16_MAX(65535);
2293
2294 if (Use32BitInsts) {
2295 assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2296, __PRETTY_FUNCTION__))
2296 "No set bits in mask when using 32-bit ands for 64-bit value")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value"
) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2296, __PRETTY_FUNCTION__))
;
2297
2298 if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2299 (unsigned) (ANDISMask != 0) +
2300 (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2301
2302 SDValue ANDIVal, ANDISVal;
2303 if (ANDIMask != 0)
2304 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo8, dl, MVT::i64,
2305 ExtendToInt64(Res, dl), getI32Imm(ANDIMask, dl)), 0);
2306 if (ANDISMask != 0)
2307 ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo8, dl, MVT::i64,
2308 ExtendToInt64(Res, dl), getI32Imm(ANDISMask, dl)), 0);
2309
2310 if (!ANDIVal)
2311 Res = ANDISVal;
2312 else if (!ANDISVal)
2313 Res = ANDIVal;
2314 else
2315 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2316 ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2317 } else {
2318 if (InstCnt) *InstCnt += selectI64ImmInstrCount(Mask) + /* and */ 1;
2319
2320 SDValue MaskVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0);
2321 Res =
2322 SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2323 ExtendToInt64(Res, dl), MaskVal), 0);
2324 }
2325 }
2326
2327 return Res.getNode();
2328 }
2329
2330 SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) {
2331 // Fill in BitGroups.
2332 collectBitGroups(LateMask);
2333 if (BitGroups.empty())
2334 return nullptr;
2335
2336 // For 64-bit values, figure out when we can use 32-bit instructions.
2337 if (Bits.size() == 64)
2338 assignRepl32BitGroups();
2339
2340 // Fill in ValueRotsVec.
2341 collectValueRotInfo();
2342
2343 if (Bits.size() == 32) {
2344 return Select32(N, LateMask, InstCnt);
2345 } else {
2346 assert(Bits.size() == 64 && "Not 64 bits here?")((Bits.size() == 64 && "Not 64 bits here?") ? static_cast
<void> (0) : __assert_fail ("Bits.size() == 64 && \"Not 64 bits here?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2346, __PRETTY_FUNCTION__))
;
2347 return Select64(N, LateMask, InstCnt);
2348 }
2349
2350 return nullptr;
2351 }
2352
2353 void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) {
2354 BitGroups.erase(remove_if(BitGroups, F), BitGroups.end());
2355 }
2356
2357 SmallVector<ValueBit, 64> Bits;
2358
2359 bool NeedMask;
2360 SmallVector<unsigned, 64> RLAmt;
2361
2362 SmallVector<BitGroup, 16> BitGroups;
2363
2364 DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots;
2365 SmallVector<ValueRotInfo, 16> ValueRotsVec;
2366
2367 SelectionDAG *CurDAG;
2368
2369public:
2370 BitPermutationSelector(SelectionDAG *DAG)
2371 : CurDAG(DAG) {}
2372
2373 // Here we try to match complex bit permutations into a set of
2374 // rotate-and-shift/shift/and/or instructions, using a set of heuristics
2375 // known to produce optimal code for common cases (like i32 byte swapping).
2376 SDNode *Select(SDNode *N) {
2377 Memoizer.clear();
2378 auto Result =
2379 getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits());
2380 if (!Result.first)
2381 return nullptr;
2382 Bits = std::move(*Result.second);
2383
2384 LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Considering bit-permutation-based instruction"
" selection for: "; } } while (false)
2385 " selection for: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Considering bit-permutation-based instruction"
" selection for: "; } } while (false)
;
2386 LLVM_DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { N->dump(CurDAG); } } while (false)
;
2387
2388 // Fill it RLAmt and set NeedMask.
2389 computeRotationAmounts();
2390
2391 if (!NeedMask)
2392 return Select(N, false);
2393
2394 // We currently have two techniques for handling results with zeros: early
2395 // masking (the default) and late masking. Late masking is sometimes more
2396 // efficient, but because the structure of the bit groups is different, it
2397 // is hard to tell without generating both and comparing the results. With
2398 // late masking, we ignore zeros in the resulting value when inserting each
2399 // set of bit groups, and then mask in the zeros at the end. With early
2400 // masking, we only insert the non-zero parts of the result at every step.
2401
2402 unsigned InstCnt = 0, InstCntLateMask = 0;
2403 LLVM_DEBUG(dbgs() << "\tEarly masking:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tEarly masking:\n"; } } while
(false)
;
2404 SDNode *RN = Select(N, false, &InstCnt);
2405 LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\tisel would use " <<
InstCnt << " instructions\n"; } } while (false)
;
2406
2407 LLVM_DEBUG(dbgs() << "\tLate masking:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tLate masking:\n"; } } while
(false)
;
2408 SDNode *RNLM = Select(N, true, &InstCntLateMask);
2409 LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMaskdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\tisel would use " <<
InstCntLateMask << " instructions\n"; } } while (false
)
2410 << " instructions\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\t\tisel would use " <<
InstCntLateMask << " instructions\n"; } } while (false
)
;
2411
2412 if (InstCnt <= InstCntLateMask) {
2413 LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tUsing early-masking for isel\n"
; } } while (false)
;
2414 return RN;
2415 }
2416
2417 LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\tUsing late-masking for isel\n"
; } } while (false)
;
2418 return RNLM;
2419 }
2420};
2421
2422class IntegerCompareEliminator {
2423 SelectionDAG *CurDAG;
2424 PPCDAGToDAGISel *S;
2425 // Conversion type for interpreting results of a 32-bit instruction as
2426 // a 64-bit value or vice versa.
2427 enum ExtOrTruncConversion { Ext, Trunc };
2428
2429 // Modifiers to guide how an ISD::SETCC node's result is to be computed
2430 // in a GPR.
2431 // ZExtOrig - use the original condition code, zero-extend value
2432 // ZExtInvert - invert the condition code, zero-extend value
2433 // SExtOrig - use the original condition code, sign-extend value
2434 // SExtInvert - invert the condition code, sign-extend value
2435 enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert };
2436
2437 // Comparisons against zero to emit GPR code sequences for. Each of these
2438 // sequences may need to be emitted for two or more equivalent patterns.
2439 // For example (a >= 0) == (a > -1). The direction of the comparison (</>)
2440 // matters as well as the extension type: sext (-1/0), zext (1/0).
2441 // GEZExt - (zext (LHS >= 0))
2442 // GESExt - (sext (LHS >= 0))
2443 // LEZExt - (zext (LHS <= 0))
2444 // LESExt - (sext (LHS <= 0))
2445 enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt };
2446
2447 SDNode *tryEXTEND(SDNode *N);
2448 SDNode *tryLogicOpOfCompares(SDNode *N);
2449 SDValue computeLogicOpInGPR(SDValue LogicOp);
2450 SDValue signExtendInputIfNeeded(SDValue Input);
2451 SDValue zeroExtendInputIfNeeded(SDValue Input);
2452 SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv);
2453 SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2454 ZeroCompare CmpTy);
2455 SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2456 int64_t RHSValue, SDLoc dl);
2457 SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2458 int64_t RHSValue, SDLoc dl);
2459 SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2460 int64_t RHSValue, SDLoc dl);
2461 SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2462 int64_t RHSValue, SDLoc dl);
2463 SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts);
2464
2465public:
2466 IntegerCompareEliminator(SelectionDAG *DAG,
2467 PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) {
2468 assert(CurDAG->getTargetLoweringInfo()((CurDAG->getTargetLoweringInfo() .getPointerTy(CurDAG->
getDataLayout()).getSizeInBits() == 64 && "Only expecting to use this on 64 bit targets."
) ? static_cast<void> (0) : __assert_fail ("CurDAG->getTargetLoweringInfo() .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && \"Only expecting to use this on 64 bit targets.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2470, __PRETTY_FUNCTION__))
2469 .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 &&((CurDAG->getTargetLoweringInfo() .getPointerTy(CurDAG->
getDataLayout()).getSizeInBits() == 64 && "Only expecting to use this on 64 bit targets."
) ? static_cast<void> (0) : __assert_fail ("CurDAG->getTargetLoweringInfo() .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && \"Only expecting to use this on 64 bit targets.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2470, __PRETTY_FUNCTION__))
2470 "Only expecting to use this on 64 bit targets.")((CurDAG->getTargetLoweringInfo() .getPointerTy(CurDAG->
getDataLayout()).getSizeInBits() == 64 && "Only expecting to use this on 64 bit targets."
) ? static_cast<void> (0) : __assert_fail ("CurDAG->getTargetLoweringInfo() .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && \"Only expecting to use this on 64 bit targets.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2470, __PRETTY_FUNCTION__))
;
2471 }
2472 SDNode *Select(SDNode *N) {
2473 if (CmpInGPR == ICGPR_None)
2474 return nullptr;
2475 switch (N->getOpcode()) {
2476 default: break;
2477 case ISD::ZERO_EXTEND:
2478 if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 ||
2479 CmpInGPR == ICGPR_SextI64)
2480 return nullptr;
2481 LLVM_FALLTHROUGH[[clang::fallthrough]];
2482 case ISD::SIGN_EXTEND:
2483 if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 ||
2484 CmpInGPR == ICGPR_ZextI64)
2485 return nullptr;
2486 return tryEXTEND(N);
2487 case ISD::AND:
2488 case ISD::OR:
2489 case ISD::XOR:
2490 return tryLogicOpOfCompares(N);
2491 }
2492 return nullptr;
2493 }
2494};
2495
2496static bool isLogicOp(unsigned Opc) {
2497 return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR;
2498}
2499// The obvious case for wanting to keep the value in a GPR. Namely, the
2500// result of the comparison is actually needed in a GPR.
2501SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) {
2502 assert((N->getOpcode() == ISD::ZERO_EXTEND ||(((N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode()
== ISD::SIGN_EXTEND) && "Expecting a zero/sign extend node!"
) ? static_cast<void> (0) : __assert_fail ("(N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) && \"Expecting a zero/sign extend node!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2504, __PRETTY_FUNCTION__))
2503 N->getOpcode() == ISD::SIGN_EXTEND) &&(((N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode()
== ISD::SIGN_EXTEND) && "Expecting a zero/sign extend node!"
) ? static_cast<void> (0) : __assert_fail ("(N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) && \"Expecting a zero/sign extend node!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2504, __PRETTY_FUNCTION__))
2504 "Expecting a zero/sign extend node!")(((N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode()
== ISD::SIGN_EXTEND) && "Expecting a zero/sign extend node!"
) ? static_cast<void> (0) : __assert_fail ("(N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) && \"Expecting a zero/sign extend node!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2504, __PRETTY_FUNCTION__))
;
2505 SDValue WideRes;
2506 // If we are zero-extending the result of a logical operation on i1
2507 // values, we can keep the values in GPRs.
2508 if (isLogicOp(N->getOperand(0).getOpcode()) &&
2509 N->getOperand(0).getValueType() == MVT::i1 &&
2510 N->getOpcode() == ISD::ZERO_EXTEND)
2511 WideRes = computeLogicOpInGPR(N->getOperand(0));
2512 else if (N->getOperand(0).getOpcode() != ISD::SETCC)
2513 return nullptr;
2514 else
2515 WideRes =
2516 getSETCCInGPR(N->getOperand(0),
2517 N->getOpcode() == ISD::SIGN_EXTEND ?
2518 SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig);
2519
2520 if (!WideRes)
2521 return nullptr;
2522
2523 SDLoc dl(N);
2524 bool Input32Bit = WideRes.getValueType() == MVT::i32;
2525 bool Output32Bit = N->getValueType(0) == MVT::i32;
2526
2527 NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0;
2528 NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1;
2529
2530 SDValue ConvOp = WideRes;
2531 if (Input32Bit != Output32Bit)
2532 ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext :
2533 ExtOrTruncConversion::Trunc);
2534 return ConvOp.getNode();
2535}
2536
2537// Attempt to perform logical operations on the results of comparisons while
2538// keeping the values in GPRs. Without doing so, these would end up being
2539// lowered to CR-logical operations which suffer from significant latency and
2540// low ILP.
2541SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) {
2542 if (N->getValueType(0) != MVT::i1)
2543 return nullptr;
2544 assert(isLogicOp(N->getOpcode()) &&((isLogicOp(N->getOpcode()) && "Expected a logic operation on setcc results."
) ? static_cast<void> (0) : __assert_fail ("isLogicOp(N->getOpcode()) && \"Expected a logic operation on setcc results.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2545, __PRETTY_FUNCTION__))
2545 "Expected a logic operation on setcc results.")((isLogicOp(N->getOpcode()) && "Expected a logic operation on setcc results."
) ? static_cast<void> (0) : __assert_fail ("isLogicOp(N->getOpcode()) && \"Expected a logic operation on setcc results.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2545, __PRETTY_FUNCTION__))
;
2546 SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0));
2547 if (!LoweredLogical)
2548 return nullptr;
2549
2550 SDLoc dl(N);
2551 bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8;
2552 unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt;
2553 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
2554 SDValue LHS = LoweredLogical.getOperand(0);
2555 SDValue RHS = LoweredLogical.getOperand(1);
2556 SDValue WideOp;
2557 SDValue OpToConvToRecForm;
2558
2559 // Look through any 32-bit to 64-bit implicit extend nodes to find the
2560 // opcode that is input to the XORI.
2561 if (IsBitwiseNegate &&
2562 LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG)
2563 OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1);
2564 else if (IsBitwiseNegate)
2565 // If the input to the XORI isn't an extension, that's what we're after.
2566 OpToConvToRecForm = LoweredLogical.getOperand(0);
2567 else
2568 // If this is not an XORI, it is a reg-reg logical op and we can convert
2569 // it to record-form.
2570 OpToConvToRecForm = LoweredLogical;
2571
2572 // Get the record-form version of the node we're looking to use to get the
2573 // CR result from.
2574 uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode();
2575 int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc);
2576
2577 // Convert the right node to record-form. This is either the logical we're
2578 // looking at or it is the input node to the negation (if we're looking at
2579 // a bitwise negation).
2580 if (NewOpc != -1 && IsBitwiseNegate) {
2581 // The input to the XORI has a record-form. Use it.
2582 assert(LoweredLogical.getConstantOperandVal(1) == 1 &&((LoweredLogical.getConstantOperandVal(1) == 1 && "Expected a PPC::XORI8 only for bitwise negation."
) ? static_cast<void> (0) : __assert_fail ("LoweredLogical.getConstantOperandVal(1) == 1 && \"Expected a PPC::XORI8 only for bitwise negation.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2583, __PRETTY_FUNCTION__))
2583 "Expected a PPC::XORI8 only for bitwise negation.")((LoweredLogical.getConstantOperandVal(1) == 1 && "Expected a PPC::XORI8 only for bitwise negation."
) ? static_cast<void> (0) : __assert_fail ("LoweredLogical.getConstantOperandVal(1) == 1 && \"Expected a PPC::XORI8 only for bitwise negation.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2583, __PRETTY_FUNCTION__))
;
2584 // Emit the record-form instruction.
2585 std::vector<SDValue> Ops;
2586 for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++)
2587 Ops.push_back(OpToConvToRecForm.getOperand(i));
2588
2589 WideOp =
2590 SDValue(CurDAG->getMachineNode(NewOpc, dl,
2591 OpToConvToRecForm.getValueType(),
2592 MVT::Glue, Ops), 0);
2593 } else {
2594 assert((NewOpc != -1 || !IsBitwiseNegate) &&(((NewOpc != -1 || !IsBitwiseNegate) && "No record form available for AND8/OR8/XOR8?"
) ? static_cast<void> (0) : __assert_fail ("(NewOpc != -1 || !IsBitwiseNegate) && \"No record form available for AND8/OR8/XOR8?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2595, __PRETTY_FUNCTION__))
2595 "No record form available for AND8/OR8/XOR8?")(((NewOpc != -1 || !IsBitwiseNegate) && "No record form available for AND8/OR8/XOR8?"
) ? static_cast<void> (0) : __assert_fail ("(NewOpc != -1 || !IsBitwiseNegate) && \"No record form available for AND8/OR8/XOR8?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2595, __PRETTY_FUNCTION__))
;
2596 WideOp =
2597 SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDIo8 : NewOpc, dl,
2598 MVT::i64, MVT::Glue, LHS, RHS), 0);
2599 }
2600
2601 // Select this node to a single bit from CR0 set by the record-form node
2602 // just created. For bitwise negation, use the EQ bit which is the equivalent
2603 // of negating the result (i.e. it is a bit set when the result of the
2604 // operation is zero).
2605 SDValue SRIdxVal =
2606 CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32);
2607 SDValue CRBit =
2608 SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
2609 MVT::i1, CR0Reg, SRIdxVal,
2610 WideOp.getValue(1)), 0);
2611 return CRBit.getNode();
2612}
2613
2614// Lower a logical operation on i1 values into a GPR sequence if possible.
2615// The result can be kept in a GPR if requested.
2616// Three types of inputs can be handled:
2617// - SETCC
2618// - TRUNCATE
2619// - Logical operation (AND/OR/XOR)
2620// There is also a special case that is handled (namely a complement operation
2621// achieved with xor %a, -1).
2622SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) {
2623 assert(isLogicOp(LogicOp.getOpcode()) &&((isLogicOp(LogicOp.getOpcode()) && "Can only handle logic operations here."
) ? static_cast<void> (0) : __assert_fail ("isLogicOp(LogicOp.getOpcode()) && \"Can only handle logic operations here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2624, __PRETTY_FUNCTION__))
2624 "Can only handle logic operations here.")((isLogicOp(LogicOp.getOpcode()) && "Can only handle logic operations here."
) ? static_cast<void> (0) : __assert_fail ("isLogicOp(LogicOp.getOpcode()) && \"Can only handle logic operations here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2624, __PRETTY_FUNCTION__))
;
2625 assert(LogicOp.getValueType() == MVT::i1 &&((LogicOp.getValueType() == MVT::i1 && "Can only handle logic operations on i1 values here."
) ? static_cast<void> (0) : __assert_fail ("LogicOp.getValueType() == MVT::i1 && \"Can only handle logic operations on i1 values here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2626, __PRETTY_FUNCTION__))
2626 "Can only handle logic operations on i1 values here.")((LogicOp.getValueType() == MVT::i1 && "Can only handle logic operations on i1 values here."
) ? static_cast<void> (0) : __assert_fail ("LogicOp.getValueType() == MVT::i1 && \"Can only handle logic operations on i1 values here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2626, __PRETTY_FUNCTION__))
;
2627 SDLoc dl(LogicOp);
2628 SDValue LHS, RHS;
2629
2630 // Special case: xor %a, -1
2631 bool IsBitwiseNegation = isBitwiseNot(LogicOp);
2632
2633 // Produces a GPR sequence for each operand of the binary logic operation.
2634 // For SETCC, it produces the respective comparison, for TRUNCATE it truncates
2635 // the value in a GPR and for logic operations, it will recursively produce
2636 // a GPR sequence for the operation.
2637 auto getLogicOperand = [&] (SDValue Operand) -> SDValue {
2638 unsigned OperandOpcode = Operand.getOpcode();
2639 if (OperandOpcode == ISD::SETCC)
2640 return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig);
2641 else if (OperandOpcode == ISD::TRUNCATE) {
2642 SDValue InputOp = Operand.getOperand(0);
2643 EVT InVT = InputOp.getValueType();
2644 return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 :
2645 PPC::RLDICL, dl, InVT, InputOp,
2646 S->getI64Imm(0, dl),
2647 S->getI64Imm(63, dl)), 0);
2648 } else if (isLogicOp(OperandOpcode))
2649 return computeLogicOpInGPR(Operand);
2650 return SDValue();
2651 };
2652 LHS = getLogicOperand(LogicOp.getOperand(0));
2653 RHS = getLogicOperand(LogicOp.getOperand(1));
2654
2655 // If a GPR sequence can't be produced for the LHS we can't proceed.
2656 // Not producing a GPR sequence for the RHS is only a problem if this isn't
2657 // a bitwise negation operation.
2658 if (!LHS || (!RHS && !IsBitwiseNegation))
2659 return SDValue();
2660
2661 NumLogicOpsOnComparison++;
2662
2663 // We will use the inputs as 64-bit values.
2664 if (LHS.getValueType() == MVT::i32)
2665 LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext);
2666 if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32)
2667 RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext);
2668
2669 unsigned NewOpc;
2670 switch (LogicOp.getOpcode()) {
2671 default: llvm_unreachable("Unknown logic operation.")::llvm::llvm_unreachable_internal("Unknown logic operation.",
"/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2671)
;
2672 case ISD::AND: NewOpc = PPC::AND8; break;
2673 case ISD::OR: NewOpc = PPC::OR8; break;
2674 case ISD::XOR: NewOpc = PPC::XOR8; break;
2675 }
2676
2677 if (IsBitwiseNegation) {
2678 RHS = S->getI64Imm(1, dl);
2679 NewOpc = PPC::XORI8;
2680 }
2681
2682 return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0);
2683
2684}
2685
2686/// If the value isn't guaranteed to be sign-extended to 64-bits, extend it.
2687/// Otherwise just reinterpret it as a 64-bit value.
2688/// Useful when emitting comparison code for 32-bit values without using
2689/// the compare instruction (which only considers the lower 32-bits).
2690SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) {
2691 assert(Input.getValueType() == MVT::i32 &&((Input.getValueType() == MVT::i32 && "Can only sign-extend 32-bit values here."
) ? static_cast<void> (0) : __assert_fail ("Input.getValueType() == MVT::i32 && \"Can only sign-extend 32-bit values here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2692, __PRETTY_FUNCTION__))
2692 "Can only sign-extend 32-bit values here.")((Input.getValueType() == MVT::i32 && "Can only sign-extend 32-bit values here."
) ? static_cast<void> (0) : __assert_fail ("Input.getValueType() == MVT::i32 && \"Can only sign-extend 32-bit values here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2692, __PRETTY_FUNCTION__))
;
2693 unsigned Opc = Input.getOpcode();
2694
2695 // The value was sign extended and then truncated to 32-bits. No need to
2696 // sign extend it again.
2697 if (Opc == ISD::TRUNCATE &&
2698 (Input.getOperand(0).getOpcode() == ISD::AssertSext ||
2699 Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND))
2700 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2701
2702 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
2703 // The input is a sign-extending load. All ppc sign-extending loads
2704 // sign-extend to the full 64-bits.
2705 if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD)
2706 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2707
2708 ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
2709 // We don't sign-extend constants.
2710 if (InputConst)
2711 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2712
2713 SDLoc dl(Input);
2714 SignExtensionsAdded++;
2715 return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl,
2716 MVT::i64, Input), 0);
2717}
2718
2719/// If the value isn't guaranteed to be zero-extended to 64-bits, extend it.
2720/// Otherwise just reinterpret it as a 64-bit value.
2721/// Useful when emitting comparison code for 32-bit values without using
2722/// the compare instruction (which only considers the lower 32-bits).
2723SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) {
2724 assert(Input.getValueType() == MVT::i32 &&((Input.getValueType() == MVT::i32 && "Can only zero-extend 32-bit values here."
) ? static_cast<void> (0) : __assert_fail ("Input.getValueType() == MVT::i32 && \"Can only zero-extend 32-bit values here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2725, __PRETTY_FUNCTION__))
2725 "Can only zero-extend 32-bit values here.")((Input.getValueType() == MVT::i32 && "Can only zero-extend 32-bit values here."
) ? static_cast<void> (0) : __assert_fail ("Input.getValueType() == MVT::i32 && \"Can only zero-extend 32-bit values here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2725, __PRETTY_FUNCTION__))
;
2726 unsigned Opc = Input.getOpcode();
2727
2728 // The only condition under which we can omit the actual extend instruction:
2729 // - The value is a positive constant
2730 // - The value comes from a load that isn't a sign-extending load
2731 // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext.
2732 bool IsTruncateOfZExt = Opc == ISD::TRUNCATE &&
2733 (Input.getOperand(0).getOpcode() == ISD::AssertZext ||
2734 Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND);
2735 if (IsTruncateOfZExt)
2736 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2737
2738 ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
2739 if (InputConst && InputConst->getSExtValue() >= 0)
2740 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2741
2742 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
2743 // The input is a load that doesn't sign-extend (it will be zero-extended).
2744 if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD)
2745 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2746
2747 // None of the above, need to zero-extend.
2748 SDLoc dl(Input);
2749 ZeroExtensionsAdded++;
2750 return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input,
2751 S->getI64Imm(0, dl),
2752 S->getI64Imm(32, dl)), 0);
2753}
2754
2755// Handle a 32-bit value in a 64-bit register and vice-versa. These are of
2756// course not actual zero/sign extensions that will generate machine code,
2757// they're just a way to reinterpret a 32 bit value in a register as a
2758// 64 bit value and vice-versa.
2759SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes,
2760 ExtOrTruncConversion Conv) {
2761 SDLoc dl(NatWidthRes);
2762
2763 // For reinterpreting 32-bit values as 64 bit values, we generate
2764 // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1>
2765 if (Conv == ExtOrTruncConversion::Ext) {
2766 SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0);
2767 SDValue SubRegIdx =
2768 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
2769 return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64,
2770 ImDef, NatWidthRes, SubRegIdx), 0);
2771 }
2772
2773 assert(Conv == ExtOrTruncConversion::Trunc &&((Conv == ExtOrTruncConversion::Trunc && "Unknown convertion between 32 and 64 bit values."
) ? static_cast<void> (0) : __assert_fail ("Conv == ExtOrTruncConversion::Trunc && \"Unknown convertion between 32 and 64 bit values.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2774, __PRETTY_FUNCTION__))
2774 "Unknown convertion between 32 and 64 bit values.")((Conv == ExtOrTruncConversion::Trunc && "Unknown convertion between 32 and 64 bit values."
) ? static_cast<void> (0) : __assert_fail ("Conv == ExtOrTruncConversion::Trunc && \"Unknown convertion between 32 and 64 bit values.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2774, __PRETTY_FUNCTION__))
;
2775 // For reinterpreting 64-bit values as 32-bit values, we just need to
2776 // EXTRACT_SUBREG (i.e. extract the low word).
2777 SDValue SubRegIdx =
2778 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
2779 return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32,
2780 NatWidthRes, SubRegIdx), 0);
2781}
2782
2783// Produce a GPR sequence for compound comparisons (<=, >=) against zero.
2784// Handle both zero-extensions and sign-extensions.
2785SDValue
2786IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2787 ZeroCompare CmpTy) {
2788 EVT InVT = LHS.getValueType();
2789 bool Is32Bit = InVT == MVT::i32;
2790 SDValue ToExtend;
2791
2792 // Produce the value that needs to be either zero or sign extended.
2793 switch (CmpTy) {
2794 case ZeroCompare::GEZExt:
2795 case ZeroCompare::GESExt:
2796 ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8,
2797 dl, InVT, LHS, LHS), 0);
2798 break;
2799 case ZeroCompare::LEZExt:
2800 case ZeroCompare::LESExt: {
2801 if (Is32Bit) {
2802 // Upper 32 bits cannot be undefined for this sequence.
2803 LHS = signExtendInputIfNeeded(LHS);
2804 SDValue Neg =
2805 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
2806 ToExtend =
2807 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
2808 Neg, S->getI64Imm(1, dl),
2809 S->getI64Imm(63, dl)), 0);
2810 } else {
2811 SDValue Addi =
2812 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
2813 S->getI64Imm(~0ULL, dl)), 0);
2814 ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2815 Addi, LHS), 0);
2816 }
2817 break;
2818 }
2819 }
2820
2821 // For 64-bit sequences, the extensions are the same for the GE/LE cases.
2822 if (!Is32Bit &&
2823 (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt))
2824 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
2825 ToExtend, S->getI64Imm(1, dl),
2826 S->getI64Imm(63, dl)), 0);
2827 if (!Is32Bit &&
2828 (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt))
2829 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend,
2830 S->getI64Imm(63, dl)), 0);
2831
2832 assert(Is32Bit && "Should have handled the 32-bit sequences above.")((Is32Bit && "Should have handled the 32-bit sequences above."
) ? static_cast<void> (0) : __assert_fail ("Is32Bit && \"Should have handled the 32-bit sequences above.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2832, __PRETTY_FUNCTION__))
;
2833 // For 32-bit sequences, the extensions differ between GE/LE cases.
2834 switch (CmpTy) {
2835 case ZeroCompare::GEZExt: {
2836 SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
2837 S->getI32Imm(31, dl) };
2838 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2839 ShiftOps), 0);
2840 }
2841 case ZeroCompare::GESExt:
2842 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend,
2843 S->getI32Imm(31, dl)), 0);
2844 case ZeroCompare::LEZExt:
2845 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend,
2846 S->getI32Imm(1, dl)), 0);
2847 case ZeroCompare::LESExt:
2848 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend,
2849 S->getI32Imm(-1, dl)), 0);
2850 }
2851
2852 // The above case covers all the enumerators so it can't have a default clause
2853 // to avoid compiler warnings.
2854 llvm_unreachable("Unknown zero-comparison type.")::llvm::llvm_unreachable_internal("Unknown zero-comparison type."
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 2854)
;
2855}
2856
2857/// Produces a zero-extended result of comparing two 32-bit values according to
2858/// the passed condition code.
2859SDValue
2860IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS,
2861 ISD::CondCode CC,
2862 int64_t RHSValue, SDLoc dl) {
2863 if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
2864 CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Sext)
2865 return SDValue();
2866 bool IsRHSZero = RHSValue == 0;
2867 bool IsRHSOne = RHSValue == 1;
2868 bool IsRHSNegOne = RHSValue == -1LL;
2869 switch (CC) {
2870 default: return SDValue();
2871 case ISD::SETEQ: {
2872 // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5)
2873 // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5)
2874 SDValue Xor = IsRHSZero ? LHS :
2875 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
2876 SDValue Clz =
2877 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
2878 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
2879 S->getI32Imm(31, dl) };
2880 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2881 ShiftOps), 0);
2882 }
2883 case ISD::SETNE: {
2884 // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1)
2885 // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1)
2886 SDValue Xor = IsRHSZero ? LHS :
2887 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
2888 SDValue Clz =
2889 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
2890 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
2891 S->getI32Imm(31, dl) };
2892 SDValue Shift =
2893 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
2894 return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
2895 S->getI32Imm(1, dl)), 0);
2896 }
2897 case ISD::SETGE: {
2898 // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1)
2899 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31)
2900 if(IsRHSZero)
2901 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
2902
2903 // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
2904 // by swapping inputs and falling through.
2905 std::swap(LHS, RHS);
2906 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
2907 IsRHSZero = RHSConst && RHSConst->isNullValue();
2908 LLVM_FALLTHROUGH[[clang::fallthrough]];
2909 }
2910 case ISD::SETLE: {
2911 if (CmpInGPR == ICGPR_NonExtIn)
2912 return SDValue();
2913 // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1)
2914 // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1)
2915 if(IsRHSZero) {
2916 if (CmpInGPR == ICGPR_NonExtIn)
2917 return SDValue();
2918 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
2919 }
2920
2921 // The upper 32-bits of the register can't be undefined for this sequence.
2922 LHS = signExtendInputIfNeeded(LHS);
2923 RHS = signExtendInputIfNeeded(RHS);
2924 SDValue Sub =
2925 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
2926 SDValue Shift =
2927 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub,
2928 S->getI64Imm(1, dl), S->getI64Imm(63, dl)),
2929 0);
2930 return
2931 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl,
2932 MVT::i64, Shift, S->getI32Imm(1, dl)), 0);
2933 }
2934 case ISD::SETGT: {
2935 // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63)
2936 // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31)
2937 // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63)
2938 // Handle SETLT -1 (which is equivalent to SETGE 0).
2939 if (IsRHSNegOne)
2940 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
2941
2942 if (IsRHSZero) {
2943 if (CmpInGPR == ICGPR_NonExtIn)
2944 return SDValue();
2945 // The upper 32-bits of the register can't be undefined for this sequence.
2946 LHS = signExtendInputIfNeeded(LHS);
2947 RHS = signExtendInputIfNeeded(RHS);
2948 SDValue Neg =
2949 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
2950 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
2951 Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0);
2952 }
2953 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
2954 // (%b < %a) by swapping inputs and falling through.
2955 std::swap(LHS, RHS);
2956 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
2957 IsRHSZero = RHSConst && RHSConst->isNullValue();
2958 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
2959 LLVM_FALLTHROUGH[[clang::fallthrough]];
2960 }
2961 case ISD::SETLT: {
2962 // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63)
2963 // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1)
2964 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31)
2965 // Handle SETLT 1 (which is equivalent to SETLE 0).
2966 if (IsRHSOne) {
2967 if (CmpInGPR == ICGPR_NonExtIn)
2968 return SDValue();
2969 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
2970 }
2971
2972 if (IsRHSZero) {
2973 SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
2974 S->getI32Imm(31, dl) };
2975 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2976 ShiftOps), 0);
2977 }
2978
2979 if (CmpInGPR == ICGPR_NonExtIn)
2980 return SDValue();
2981 // The upper 32-bits of the register can't be undefined for this sequence.
2982 LHS = signExtendInputIfNeeded(LHS);
2983 RHS = signExtendInputIfNeeded(RHS);
2984 SDValue SUBFNode =
2985 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
2986 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
2987 SUBFNode, S->getI64Imm(1, dl),
2988 S->getI64Imm(63, dl)), 0);
2989 }
2990 case ISD::SETUGE:
2991 // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1)
2992 // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1)
2993 std::swap(LHS, RHS);
2994 LLVM_FALLTHROUGH[[clang::fallthrough]];
2995 case ISD::SETULE: {
2996 if (CmpInGPR == ICGPR_NonExtIn)
2997 return SDValue();
2998 // The upper 32-bits of the register can't be undefined for this sequence.
2999 LHS = zeroExtendInputIfNeeded(LHS);
3000 RHS = zeroExtendInputIfNeeded(RHS);
3001 SDValue Subtract =
3002 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3003 SDValue SrdiNode =
3004 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3005 Subtract, S->getI64Imm(1, dl),
3006 S->getI64Imm(63, dl)), 0);
3007 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode,
3008 S->getI32Imm(1, dl)), 0);
3009 }
3010 case ISD::SETUGT:
3011 // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63)
3012 // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63)
3013 std::swap(LHS, RHS);
3014 LLVM_FALLTHROUGH[[clang::fallthrough]];
3015 case ISD::SETULT: {
3016 if (CmpInGPR == ICGPR_NonExtIn)
3017 return SDValue();
3018 // The upper 32-bits of the register can't be undefined for this sequence.
3019 LHS = zeroExtendInputIfNeeded(LHS);
3020 RHS = zeroExtendInputIfNeeded(RHS);
3021 SDValue Subtract =
3022 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3023 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3024 Subtract, S->getI64Imm(1, dl),
3025 S->getI64Imm(63, dl)), 0);
3026 }
3027 }
3028}
3029
3030/// Produces a sign-extended result of comparing two 32-bit values according to
3031/// the passed condition code.
3032SDValue
3033IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS,
3034 ISD::CondCode CC,
3035 int64_t RHSValue, SDLoc dl) {
3036 if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
3037 CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Zext)
3038 return SDValue();
3039 bool IsRHSZero = RHSValue == 0;
3040 bool IsRHSOne = RHSValue == 1;
3041 bool IsRHSNegOne = RHSValue == -1LL;
3042
3043 switch (CC) {
3044 default: return SDValue();
3045 case ISD::SETEQ: {
3046 // (sext (setcc %a, %b, seteq)) ->
3047 // (ashr (shl (ctlz (xor %a, %b)), 58), 63)
3048 // (sext (setcc %a, 0, seteq)) ->
3049 // (ashr (shl (ctlz %a), 58), 63)
3050 SDValue CountInput = IsRHSZero ? LHS :
3051 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3052 SDValue Cntlzw =
3053 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0);
3054 SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl),
3055 S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3056 SDValue Slwi =
3057 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0);
3058 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0);
3059 }
3060 case ISD::SETNE: {
3061 // Bitwise xor the operands, count leading zeros, shift right by 5 bits and
3062 // flip the bit, finally take 2's complement.
3063 // (sext (setcc %a, %b, setne)) ->
3064 // (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1))
3065 // Same as above, but the first xor is not needed.
3066 // (sext (setcc %a, 0, setne)) ->
3067 // (neg (xor (lshr (ctlz %a), 5), 1))
3068 SDValue Xor = IsRHSZero ? LHS :
3069 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3070 SDValue Clz =
3071 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3072 SDValue ShiftOps[] =
3073 { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3074 SDValue Shift =
3075 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3076 SDValue Xori =
3077 SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3078 S->getI32Imm(1, dl)), 0);
3079 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0);
3080 }
3081 case ISD::SETGE: {
3082 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1)
3083 // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31)
3084 if (IsRHSZero)
3085 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3086
3087 // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3088 // by swapping inputs and falling through.
3089 std::swap(LHS, RHS);
3090 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3091 IsRHSZero = RHSConst && RHSConst->isNullValue();
3092 LLVM_FALLTHROUGH[[clang::fallthrough]];
3093 }
3094 case ISD::SETLE: {
3095 if (CmpInGPR == ICGPR_NonExtIn)
3096 return SDValue();
3097 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1)
3098 // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1)
3099 if (IsRHSZero)
3100 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3101
3102 // The upper 32-bits of the register can't be undefined for this sequence.
3103 LHS = signExtendInputIfNeeded(LHS);
3104 RHS = signExtendInputIfNeeded(RHS);
3105 SDValue SUBFNode =
3106 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue,
3107 LHS, RHS), 0);
3108 SDValue Srdi =
3109 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3110 SUBFNode, S->getI64Imm(1, dl),
3111 S->getI64Imm(63, dl)), 0);
3112 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi,
3113 S->getI32Imm(-1, dl)), 0);
3114 }
3115 case ISD::SETGT: {
3116 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63)
3117 // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31)
3118 // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63)
3119 if (IsRHSNegOne)
3120 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3121 if (IsRHSZero) {
3122 if (CmpInGPR == ICGPR_NonExtIn)
3123 return SDValue();
3124 // The upper 32-bits of the register can't be undefined for this sequence.
3125 LHS = signExtendInputIfNeeded(LHS);
3126 RHS = signExtendInputIfNeeded(RHS);
3127 SDValue Neg =
3128 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3129 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg,
3130 S->getI64Imm(63, dl)), 0);
3131 }
3132 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3133 // (%b < %a) by swapping inputs and falling through.
3134 std::swap(LHS, RHS);
3135 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3136 IsRHSZero = RHSConst && RHSConst->isNullValue();
3137 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3138 LLVM_FALLTHROUGH[[clang::fallthrough]];
3139 }
3140 case ISD::SETLT: {
3141 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63)
3142 // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1)
3143 // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31)
3144 if (IsRHSOne) {
3145 if (CmpInGPR == ICGPR_NonExtIn)
3146 return SDValue();
3147 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3148 }
3149 if (IsRHSZero)
3150 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS,
3151 S->getI32Imm(31, dl)), 0);
3152
3153 if (CmpInGPR == ICGPR_NonExtIn)
3154 return SDValue();
3155 // The upper 32-bits of the register can't be undefined for this sequence.
3156 LHS = signExtendInputIfNeeded(LHS);
3157 RHS = signExtendInputIfNeeded(RHS);
3158 SDValue SUBFNode =
3159 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3160 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3161 SUBFNode, S->getI64Imm(63, dl)), 0);
3162 }
3163 case ISD::SETUGE:
3164 // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1)
3165 // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1)
3166 std::swap(LHS, RHS);
3167 LLVM_FALLTHROUGH[[clang::fallthrough]];
3168 case ISD::SETULE: {
3169 if (CmpInGPR == ICGPR_NonExtIn)
3170 return SDValue();
3171 // The upper 32-bits of the register can't be undefined for this sequence.
3172 LHS = zeroExtendInputIfNeeded(LHS);
3173 RHS = zeroExtendInputIfNeeded(RHS);
3174 SDValue Subtract =
3175 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3176 SDValue Shift =
3177 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract,
3178 S->getI32Imm(1, dl), S->getI32Imm(63,dl)),
3179 0);
3180 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift,
3181 S->getI32Imm(-1, dl)), 0);
3182 }
3183 case ISD::SETUGT:
3184 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63)
3185 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63)
3186 std::swap(LHS, RHS);
3187 LLVM_FALLTHROUGH[[clang::fallthrough]];
3188 case ISD::SETULT: {
3189 if (CmpInGPR == ICGPR_NonExtIn)
3190 return SDValue();
3191 // The upper 32-bits of the register can't be undefined for this sequence.
3192 LHS = zeroExtendInputIfNeeded(LHS);
3193 RHS = zeroExtendInputIfNeeded(RHS);
3194 SDValue Subtract =
3195 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3196 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3197 Subtract, S->getI64Imm(63, dl)), 0);
3198 }
3199 }
3200}
3201
3202/// Produces a zero-extended result of comparing two 64-bit values according to
3203/// the passed condition code.
3204SDValue
3205IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS,
3206 ISD::CondCode CC,
3207 int64_t RHSValue, SDLoc dl) {
3208 if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3209 CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Sext)
3210 return SDValue();
3211 bool IsRHSZero = RHSValue == 0;
3212 bool IsRHSOne = RHSValue == 1;
3213 bool IsRHSNegOne = RHSValue == -1LL;
3214 switch (CC) {
3215 default: return SDValue();
3216 case ISD::SETEQ: {
3217 // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6)
3218 // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6)
3219 SDValue Xor = IsRHSZero ? LHS :
3220 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3221 SDValue Clz =
3222 SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0);
3223 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz,
3224 S->getI64Imm(58, dl),
3225 S->getI64Imm(63, dl)), 0);
3226 }
3227 case ISD::SETNE: {
3228 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3229 // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA)
3230 // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3231 // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3232 SDValue Xor = IsRHSZero ? LHS :
3233 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3234 SDValue AC =
3235 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3236 Xor, S->getI32Imm(~0U, dl)), 0);
3237 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC,
3238 Xor, AC.getValue(1)), 0);
3239 }
3240 case ISD::SETGE: {
3241 // {subc.reg, subc.CA} = (subcarry %a, %b)
3242 // (zext (setcc %a, %b, setge)) ->
3243 // (adde (lshr %b, 63), (ashr %a, 63), subc.CA)
3244 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63)
3245 if (IsRHSZero)
3246 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3247 std::swap(LHS, RHS);
3248 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3249 IsRHSZero = RHSConst && RHSConst->isNullValue();
3250 LLVM_FALLTHROUGH[[clang::fallthrough]];
3251 }
3252 case ISD::SETLE: {
3253 // {subc.reg, subc.CA} = (subcarry %b, %a)
3254 // (zext (setcc %a, %b, setge)) ->
3255 // (adde (lshr %a, 63), (ashr %b, 63), subc.CA)
3256 // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63)
3257 if (IsRHSZero)
3258 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3259 SDValue ShiftL =
3260 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3261 S->getI64Imm(1, dl),
3262 S->getI64Imm(63, dl)), 0);
3263 SDValue ShiftR =
3264 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3265 S->getI64Imm(63, dl)), 0);
3266 SDValue SubtractCarry =
3267 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3268 LHS, RHS), 1);
3269 return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3270 ShiftR, ShiftL, SubtractCarry), 0);
3271 }
3272 case ISD::SETGT: {
3273 // {subc.reg, subc.CA} = (subcarry %b, %a)
3274 // (zext (setcc %a, %b, setgt)) ->
3275 // (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3276 // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63)
3277 if (IsRHSNegOne)
3278 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3279 if (IsRHSZero) {
3280 SDValue Addi =
3281 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3282 S->getI64Imm(~0ULL, dl)), 0);
3283 SDValue Nor =
3284 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0);
3285 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor,
3286 S->getI64Imm(1, dl),
3287 S->getI64Imm(63, dl)), 0);
3288 }
3289 std::swap(LHS, RHS);
3290 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3291 IsRHSZero = RHSConst && RHSConst->isNullValue();
3292 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3293 LLVM_FALLTHROUGH[[clang::fallthrough]];
3294 }
3295 case ISD::SETLT: {
3296 // {subc.reg, subc.CA} = (subcarry %a, %b)
3297 // (zext (setcc %a, %b, setlt)) ->
3298 // (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3299 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63)
3300 if (IsRHSOne)
3301 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3302 if (IsRHSZero)
3303 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3304 S->getI64Imm(1, dl),
3305 S->getI64Imm(63, dl)), 0);
3306 SDValue SRADINode =
3307 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3308 LHS, S->getI64Imm(63, dl)), 0);
3309 SDValue SRDINode =
3310 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3311 RHS, S->getI64Imm(1, dl),
3312 S->getI64Imm(63, dl)), 0);
3313 SDValue SUBFC8Carry =
3314 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3315 RHS, LHS), 1);
3316 SDValue ADDE8Node =
3317 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3318 SRDINode, SRADINode, SUBFC8Carry), 0);
3319 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3320 ADDE8Node, S->getI64Imm(1, dl)), 0);
3321 }
3322 case ISD::SETUGE:
3323 // {subc.reg, subc.CA} = (subcarry %a, %b)
3324 // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1)
3325 std::swap(LHS, RHS);
3326 LLVM_FALLTHROUGH[[clang::fallthrough]];
3327 case ISD::SETULE: {
3328 // {subc.reg, subc.CA} = (subcarry %b, %a)
3329 // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1)
3330 SDValue SUBFC8Carry =
3331 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3332 LHS, RHS), 1);
3333 SDValue SUBFE8Node =
3334 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue,
3335 LHS, LHS, SUBFC8Carry), 0);
3336 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64,
3337 SUBFE8Node, S->getI64Imm(1, dl)), 0);
3338 }
3339 case ISD::SETUGT:
3340 // {subc.reg, subc.CA} = (subcarry %b, %a)
3341 // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA)
3342 std::swap(LHS, RHS);
3343 LLVM_FALLTHROUGH[[clang::fallthrough]];
3344 case ISD::SETULT: {
3345 // {subc.reg, subc.CA} = (subcarry %a, %b)
3346 // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA)
3347 SDValue SubtractCarry =
3348 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3349 RHS, LHS), 1);
3350 SDValue ExtSub =
3351 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3352 LHS, LHS, SubtractCarry), 0);
3353 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3354 ExtSub), 0);
3355 }
3356 }
3357}
3358
3359/// Produces a sign-extended result of comparing two 64-bit values according to
3360/// the passed condition code.
3361SDValue
3362IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS,
3363 ISD::CondCode CC,
3364 int64_t RHSValue, SDLoc dl) {
3365 if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3366 CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Zext)
3367 return SDValue();
3368 bool IsRHSZero = RHSValue == 0;
3369 bool IsRHSOne = RHSValue == 1;
3370 bool IsRHSNegOne = RHSValue == -1LL;
3371 switch (CC) {
3372 default: return SDValue();
3373 case ISD::SETEQ: {
3374 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3375 // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA)
3376 // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3377 // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3378 SDValue AddInput = IsRHSZero ? LHS :
3379 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3380 SDValue Addic =
3381 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3382 AddInput, S->getI32Imm(~0U, dl)), 0);
3383 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic,
3384 Addic, Addic.getValue(1)), 0);
3385 }
3386 case ISD::SETNE: {
3387 // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b))
3388 // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA)
3389 // {subfcz.reg, subfcz.CA} = (subcarry 0, %a)
3390 // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA)
3391 SDValue Xor = IsRHSZero ? LHS :
3392 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3393 SDValue SC =
3394 SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue,
3395 Xor, S->getI32Imm(0, dl)), 0);
3396 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC,
3397 SC, SC.getValue(1)), 0);
3398 }
3399 case ISD::SETGE: {
3400 // {subc.reg, subc.CA} = (subcarry %a, %b)
3401 // (zext (setcc %a, %b, setge)) ->
3402 // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA))
3403 // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63))
3404 if (IsRHSZero)
3405 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3406 std::swap(LHS, RHS);
3407 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3408 IsRHSZero = RHSConst && RHSConst->isNullValue();
3409 LLVM_FALLTHROUGH[[clang::fallthrough]];
3410 }
3411 case ISD::SETLE: {
3412 // {subc.reg, subc.CA} = (subcarry %b, %a)
3413 // (zext (setcc %a, %b, setge)) ->
3414 // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA))
3415 // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63)
3416 if (IsRHSZero)
3417 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3418 SDValue ShiftR =
3419 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3420 S->getI64Imm(63, dl)), 0);
3421 SDValue ShiftL =
3422 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3423 S->getI64Imm(1, dl),
3424 S->getI64Imm(63, dl)), 0);
3425 SDValue SubtractCarry =
3426 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3427 LHS, RHS), 1);
3428 SDValue Adde =
3429 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3430 ShiftR, ShiftL, SubtractCarry), 0);
3431 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0);
3432 }
3433 case ISD::SETGT: {
3434 // {subc.reg, subc.CA} = (subcarry %b, %a)
3435 // (zext (setcc %a, %b, setgt)) ->
3436 // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3437 // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63)
3438 if (IsRHSNegOne)
3439 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3440 if (IsRHSZero) {
3441 SDValue Add =
3442 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3443 S->getI64Imm(-1, dl)), 0);
3444 SDValue Nor =
3445 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0);
3446 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor,
3447 S->getI64Imm(63, dl)), 0);
3448 }
3449 std::swap(LHS, RHS);
3450 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3451 IsRHSZero = RHSConst && RHSConst->isNullValue();
3452 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3453 LLVM_FALLTHROUGH[[clang::fallthrough]];
3454 }
3455 case ISD::SETLT: {
3456 // {subc.reg, subc.CA} = (subcarry %a, %b)
3457 // (zext (setcc %a, %b, setlt)) ->
3458 // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3459 // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63)
3460 if (IsRHSOne)
3461 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3462 if (IsRHSZero) {
3463 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS,
3464 S->getI64Imm(63, dl)), 0);
3465 }
3466 SDValue SRADINode =
3467 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3468 LHS, S->getI64Imm(63, dl)), 0);
3469 SDValue SRDINode =
3470 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3471 RHS, S->getI64Imm(1, dl),
3472 S->getI64Imm(63, dl)), 0);
3473 SDValue SUBFC8Carry =
3474 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3475 RHS, LHS), 1);
3476 SDValue ADDE8Node =
3477 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64,
3478 SRDINode, SRADINode, SUBFC8Carry), 0);
3479 SDValue XORI8Node =
3480 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3481 ADDE8Node, S->getI64Imm(1, dl)), 0);
3482 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3483 XORI8Node), 0);
3484 }
3485 case ISD::SETUGE:
3486 // {subc.reg, subc.CA} = (subcarry %a, %b)
3487 // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA)
3488 std::swap(LHS, RHS);
3489 LLVM_FALLTHROUGH[[clang::fallthrough]];
3490 case ISD::SETULE: {
3491 // {subc.reg, subc.CA} = (subcarry %b, %a)
3492 // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA)
3493 SDValue SubtractCarry =
3494 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3495 LHS, RHS), 1);
3496 SDValue ExtSub =
3497 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS,
3498 LHS, SubtractCarry), 0);
3499 return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64,
3500 ExtSub, ExtSub), 0);
3501 }
3502 case ISD::SETUGT:
3503 // {subc.reg, subc.CA} = (subcarry %b, %a)
3504 // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA)
3505 std::swap(LHS, RHS);
3506 LLVM_FALLTHROUGH[[clang::fallthrough]];
3507 case ISD::SETULT: {
3508 // {subc.reg, subc.CA} = (subcarry %a, %b)
3509 // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA)
3510 SDValue SubCarry =
3511 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3512 RHS, LHS), 1);
3513 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3514 LHS, LHS, SubCarry), 0);
3515 }
3516 }
3517}
3518
3519/// Do all uses of this SDValue need the result in a GPR?
3520/// This is meant to be used on values that have type i1 since
3521/// it is somewhat meaningless to ask if values of other types
3522/// should be kept in GPR's.
3523static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) {
3524 assert(Compare.getOpcode() == ISD::SETCC &&((Compare.getOpcode() == ISD::SETCC && "An ISD::SETCC node required here."
) ? static_cast<void> (0) : __assert_fail ("Compare.getOpcode() == ISD::SETCC && \"An ISD::SETCC node required here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3525, __PRETTY_FUNCTION__))
3525 "An ISD::SETCC node required here.")((Compare.getOpcode() == ISD::SETCC && "An ISD::SETCC node required here."
) ? static_cast<void> (0) : __assert_fail ("Compare.getOpcode() == ISD::SETCC && \"An ISD::SETCC node required here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3525, __PRETTY_FUNCTION__))
;
3526
3527 // For values that have a single use, the caller should obviously already have
3528 // checked if that use is an extending use. We check the other uses here.
3529 if (Compare.hasOneUse())
3530 return true;
3531 // We want the value in a GPR if it is being extended, used for a select, or
3532 // used in logical operations.
3533 for (auto CompareUse : Compare.getNode()->uses())
3534 if (CompareUse->getOpcode() != ISD::SIGN_EXTEND &&
3535 CompareUse->getOpcode() != ISD::ZERO_EXTEND &&
3536 CompareUse->getOpcode() != ISD::SELECT &&
3537 !isLogicOp(CompareUse->getOpcode())) {
3538 OmittedForNonExtendUses++;
3539 return false;
3540 }
3541 return true;
3542}
3543
3544/// Returns an equivalent of a SETCC node but with the result the same width as
3545/// the inputs. This can also be used for SELECT_CC if either the true or false
3546/// values is a power of two while the other is zero.
3547SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare,
3548 SetccInGPROpts ConvOpts) {
3549 assert((Compare.getOpcode() == ISD::SETCC ||(((Compare.getOpcode() == ISD::SETCC || Compare.getOpcode() ==
ISD::SELECT_CC) && "An ISD::SETCC node required here."
) ? static_cast<void> (0) : __assert_fail ("(Compare.getOpcode() == ISD::SETCC || Compare.getOpcode() == ISD::SELECT_CC) && \"An ISD::SETCC node required here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3551, __PRETTY_FUNCTION__))
3550 Compare.getOpcode() == ISD::SELECT_CC) &&(((Compare.getOpcode() == ISD::SETCC || Compare.getOpcode() ==
ISD::SELECT_CC) && "An ISD::SETCC node required here."
) ? static_cast<void> (0) : __assert_fail ("(Compare.getOpcode() == ISD::SETCC || Compare.getOpcode() == ISD::SELECT_CC) && \"An ISD::SETCC node required here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3551, __PRETTY_FUNCTION__))
3551 "An ISD::SETCC node required here.")(((Compare.getOpcode() == ISD::SETCC || Compare.getOpcode() ==
ISD::SELECT_CC) && "An ISD::SETCC node required here."
) ? static_cast<void> (0) : __assert_fail ("(Compare.getOpcode() == ISD::SETCC || Compare.getOpcode() == ISD::SELECT_CC) && \"An ISD::SETCC node required here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3551, __PRETTY_FUNCTION__))
;
3552
3553 // Don't convert this comparison to a GPR sequence because there are uses
3554 // of the i1 result (i.e. uses that require the result in the CR).
3555 if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG))
3556 return SDValue();
3557
3558 SDValue LHS = Compare.getOperand(0);
3559 SDValue RHS = Compare.getOperand(1);
3560
3561 // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC.
3562 int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2;
3563 ISD::CondCode CC =
3564 cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get();
3565 EVT InputVT = LHS.getValueType();
3566 if (InputVT != MVT::i32 && InputVT != MVT::i64)
3567 return SDValue();
3568
3569 if (ConvOpts == SetccInGPROpts::ZExtInvert ||
3570 ConvOpts == SetccInGPROpts::SExtInvert)
3571 CC = ISD::getSetCCInverse(CC, true);
3572
3573 bool Inputs32Bit = InputVT == MVT::i32;
3574
3575 SDLoc dl(Compare);
3576 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3577 int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX(9223372036854775807L);
3578 bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig ||
3579 ConvOpts == SetccInGPROpts::SExtInvert;
3580
3581 if (IsSext && Inputs32Bit)
3582 return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3583 else if (Inputs32Bit)
3584 return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3585 else if (IsSext)
3586 return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3587 return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3588}
3589
3590} // end anonymous namespace
3591
3592bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) {
3593 if (N->getValueType(0) != MVT::i32 &&
3594 N->getValueType(0) != MVT::i64)
3595 return false;
3596
3597 // This optimization will emit code that assumes 64-bit registers
3598 // so we don't want to run it in 32-bit mode. Also don't run it
3599 // on functions that are not to be optimized.
3600 if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64())
3601 return false;
3602
3603 switch (N->getOpcode()) {
3604 default: break;
3605 case ISD::ZERO_EXTEND:
3606 case ISD::SIGN_EXTEND:
3607 case ISD::AND:
3608 case ISD::OR:
3609 case ISD::XOR: {
3610 IntegerCompareEliminator ICmpElim(CurDAG, this);
3611 if (SDNode *New = ICmpElim.Select(N)) {
3612 ReplaceNode(N, New);
3613 return true;
3614 }
3615 }
3616 }
3617 return false;
3618}
3619
3620bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) {
3621 if (N->getValueType(0) != MVT::i32 &&
3622 N->getValueType(0) != MVT::i64)
3623 return false;
3624
3625 if (!UseBitPermRewriter)
3626 return false;
3627
3628 switch (N->getOpcode()) {
3629 default: break;
3630 case ISD::ROTL:
3631 case ISD::SHL:
3632 case ISD::SRL:
3633 case ISD::AND:
3634 case ISD::OR: {
3635 BitPermutationSelector BPS(CurDAG);
3636 if (SDNode *New = BPS.Select(N)) {
3637 ReplaceNode(N, New);
3638 return true;
3639 }
3640 return false;
3641 }
3642 }
3643
3644 return false;
3645}
3646
3647/// SelectCC - Select a comparison of the specified values with the specified
3648/// condition code, returning the CR# of the expression.
3649SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3650 const SDLoc &dl) {
3651 // Always select the LHS.
3652 unsigned Opc;
3653
3654 if (LHS.getValueType() == MVT::i32) {
3655 unsigned Imm;
3656 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
3657 if (isInt32Immediate(RHS, Imm)) {
3658 // SETEQ/SETNE comparison with 16-bit immediate, fold it.
3659 if (isUInt<16>(Imm))
3660 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3661 getI32Imm(Imm & 0xFFFF, dl)),
3662 0);
3663 // If this is a 16-bit signed immediate, fold it.
3664 if (isInt<16>((int)Imm))
3665 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3666 getI32Imm(Imm & 0xFFFF, dl)),
3667 0);
3668
3669 // For non-equality comparisons, the default code would materialize the
3670 // constant, then compare against it, like this:
3671 // lis r2, 4660
3672 // ori r2, r2, 22136
3673 // cmpw cr0, r3, r2
3674 // Since we are just comparing for equality, we can emit this instead:
3675 // xoris r0,r3,0x1234
3676 // cmplwi cr0,r0,0x5678
3677 // beq cr0,L6
3678 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS,
3679 getI32Imm(Imm >> 16, dl)), 0);
3680 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor,
3681 getI32Imm(Imm & 0xFFFF, dl)), 0);
3682 }
3683 Opc = PPC::CMPLW;
3684 } else if (ISD::isUnsignedIntSetCC(CC)) {
3685 if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm))
3686 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3687 getI32Imm(Imm & 0xFFFF, dl)), 0);
3688 Opc = PPC::CMPLW;
3689 } else {
3690 int16_t SImm;
3691 if (isIntS16Immediate(RHS, SImm))
3692 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3693 getI32Imm((int)SImm & 0xFFFF,
3694 dl)),
3695 0);
3696 Opc = PPC::CMPW;
3697 }
3698 } else if (LHS.getValueType() == MVT::i64) {
3699 uint64_t Imm;
3700 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
3701 if (isInt64Immediate(RHS.getNode(), Imm)) {
3702 // SETEQ/SETNE comparison with 16-bit immediate, fold it.
3703 if (isUInt<16>(Imm))
3704 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
3705 getI32Imm(Imm & 0xFFFF, dl)),
3706 0);
3707 // If this is a 16-bit signed immediate, fold it.
3708 if (isInt<16>(Imm))
3709 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
3710 getI32Imm(Imm & 0xFFFF, dl)),
3711 0);
3712
3713 // For non-equality comparisons, the default code would materialize the
3714 // constant, then compare against it, like this:
3715 // lis r2, 4660
3716 // ori r2, r2, 22136
3717 // cmpd cr0, r3, r2
3718 // Since we are just comparing for equality, we can emit this instead:
3719 // xoris r0,r3,0x1234
3720 // cmpldi cr0,r0,0x5678
3721 // beq cr0,L6
3722 if (isUInt<32>(Imm)) {
3723 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS,
3724 getI64Imm(Imm >> 16, dl)), 0);
3725 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor,
3726 getI64Imm(Imm & 0xFFFF, dl)),
3727 0);
3728 }
3729 }
3730 Opc = PPC::CMPLD;
3731 } else if (ISD::isUnsignedIntSetCC(CC)) {
3732 if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm))
3733 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
3734 getI64Imm(Imm & 0xFFFF, dl)), 0);
3735 Opc = PPC::CMPLD;
3736 } else {
3737 int16_t SImm;
3738 if (isIntS16Immediate(RHS, SImm))
3739 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
3740 getI64Imm(SImm & 0xFFFF, dl)),
3741 0);
3742 Opc = PPC::CMPD;
3743 }
3744 } else if (LHS.getValueType() == MVT::f32) {
3745 if (PPCSubTarget->hasSPE()) {
3746 switch (CC) {
3747 default:
3748 case ISD::SETEQ:
3749 case ISD::SETNE:
3750 Opc = PPC::EFSCMPEQ;
3751 break;
3752 case ISD::SETLT:
3753 case ISD::SETGE:
3754 case ISD::SETOLT:
3755 case ISD::SETOGE:
3756 case ISD::SETULT:
3757 case ISD::SETUGE:
3758 Opc = PPC::EFSCMPLT;
3759 break;
3760 case ISD::SETGT:
3761 case ISD::SETLE:
3762 case ISD::SETOGT:
3763 case ISD::SETOLE:
3764 case ISD::SETUGT:
3765 case ISD::SETULE:
3766 Opc = PPC::EFSCMPGT;
3767 break;
3768 }
3769 } else
3770 Opc = PPC::FCMPUS;
3771 } else if (LHS.getValueType() == MVT::f64) {
3772 if (PPCSubTarget->hasSPE()) {
3773 switch (CC) {
3774 default:
3775 case ISD::SETEQ:
3776 case ISD::SETNE:
3777 Opc = PPC::EFDCMPEQ;
3778 break;
3779 case ISD::SETLT:
3780 case ISD::SETGE:
3781 case ISD::SETOLT:
3782 case ISD::SETOGE:
3783 case ISD::SETULT:
3784 case ISD::SETUGE:
3785 Opc = PPC::EFDCMPLT;
3786 break;
3787 case ISD::SETGT:
3788 case ISD::SETLE:
3789 case ISD::SETOGT:
3790 case ISD::SETOLE:
3791 case ISD::SETUGT:
3792 case ISD::SETULE:
3793 Opc = PPC::EFDCMPGT;
3794 break;
3795 }
3796 } else
3797 Opc = PPCSubTarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
3798 } else {
3799 assert(LHS.getValueType() == MVT::f128 && "Unknown vt!")((LHS.getValueType() == MVT::f128 && "Unknown vt!") ?
static_cast<void> (0) : __assert_fail ("LHS.getValueType() == MVT::f128 && \"Unknown vt!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3799, __PRETTY_FUNCTION__))
;
3800 assert(PPCSubTarget->hasVSX() && "__float128 requires VSX")((PPCSubTarget->hasVSX() && "__float128 requires VSX"
) ? static_cast<void> (0) : __assert_fail ("PPCSubTarget->hasVSX() && \"__float128 requires VSX\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3800, __PRETTY_FUNCTION__))
;
3801 Opc = PPC::XSCMPUQP;
3802 }
3803 return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0);
3804}
3805
3806static PPC::Predicate getPredicateForSetCC(ISD::CondCode CC) {
3807 switch (CC) {
3808 case ISD::SETUEQ:
3809 case ISD::SETONE:
3810 case ISD::SETOLE:
3811 case ISD::SETOGE:
3812 llvm_unreachable("Should be lowered by legalize!")::llvm::llvm_unreachable_internal("Should be lowered by legalize!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3812)
;
3813 default: llvm_unreachable("Unknown condition!")::llvm::llvm_unreachable_internal("Unknown condition!", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3813)
;
3814 case ISD::SETOEQ:
3815 case ISD::SETEQ: return PPC::PRED_EQ;
3816 case ISD::SETUNE:
3817 case ISD::SETNE: return PPC::PRED_NE;
3818 case ISD::SETOLT:
3819 case ISD::SETLT: return PPC::PRED_LT;
3820 case ISD::SETULE:
3821 case ISD::SETLE: return PPC::PRED_LE;
3822 case ISD::SETOGT:
3823 case ISD::SETGT: return PPC::PRED_GT;
3824 case ISD::SETUGE:
3825 case ISD::SETGE: return PPC::PRED_GE;
3826 case ISD::SETO: return PPC::PRED_NU;
3827 case ISD::SETUO: return PPC::PRED_UN;
3828 // These two are invalid for floating point. Assume we have int.
3829 case ISD::SETULT: return PPC::PRED_LT;
3830 case ISD::SETUGT: return PPC::PRED_GT;
3831 }
3832}
3833
3834/// getCRIdxForSetCC - Return the index of the condition register field
3835/// associated with the SetCC condition, and whether or not the field is
3836/// treated as inverted. That is, lt = 0; ge = 0 inverted.
3837static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) {
3838 Invert = false;
3839 switch (CC) {
3840 default: llvm_unreachable("Unknown condition!")::llvm::llvm_unreachable_internal("Unknown condition!", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3840)
;
3841 case ISD::SETOLT:
3842 case ISD::SETLT: return 0; // Bit #0 = SETOLT
3843 case ISD::SETOGT:
3844 case ISD::SETGT: return 1; // Bit #1 = SETOGT
3845 case ISD::SETOEQ:
3846 case ISD::SETEQ: return 2; // Bit #2 = SETOEQ
3847 case ISD::SETUO: return 3; // Bit #3 = SETUO
3848 case ISD::SETUGE:
3849 case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE
3850 case ISD::SETULE:
3851 case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE
3852 case ISD::SETUNE:
3853 case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE
3854 case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO
3855 case ISD::SETUEQ:
3856 case ISD::SETOGE:
3857 case ISD::SETOLE:
3858 case ISD::SETONE:
3859 llvm_unreachable("Invalid branch code: should be expanded by legalize")::llvm::llvm_unreachable_internal("Invalid branch code: should be expanded by legalize"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3859)
;
3860 // These are invalid for floating point. Assume integer.
3861 case ISD::SETULT: return 0;
3862 case ISD::SETUGT: return 1;
3863 }
3864}
3865
3866// getVCmpInst: return the vector compare instruction for the specified
3867// vector type and condition code. Since this is for altivec specific code,
3868// only support the altivec types (v16i8, v8i16, v4i32, v2i64, and v4f32).
3869static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC,
3870 bool HasVSX, bool &Swap, bool &Negate) {
3871 Swap = false;
3872 Negate = false;
3873
3874 if (VecVT.isFloatingPoint()) {
3875 /* Handle some cases by swapping input operands. */
3876 switch (CC) {
3877 case ISD::SETLE: CC = ISD::SETGE; Swap = true; break;
3878 case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
3879 case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break;
3880 case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break;
3881 case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
3882 case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break;
3883 default: break;
3884 }
3885 /* Handle some cases by negating the result. */
3886 switch (CC) {
3887 case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
3888 case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break;
3889 case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break;
3890 case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break;
3891 default: break;
3892 }
3893 /* We have instructions implementing the remaining cases. */
3894 switch (CC) {
3895 case ISD::SETEQ:
3896 case ISD::SETOEQ:
3897 if (VecVT == MVT::v4f32)
3898 return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP;
3899 else if (VecVT == MVT::v2f64)
3900 return PPC::XVCMPEQDP;
3901 break;
3902 case ISD::SETGT:
3903 case ISD::SETOGT:
3904 if (VecVT == MVT::v4f32)
3905 return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP;
3906 else if (VecVT == MVT::v2f64)
3907 return PPC::XVCMPGTDP;
3908 break;
3909 case ISD::SETGE:
3910 case ISD::SETOGE:
3911 if (VecVT == MVT::v4f32)
3912 return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP;
3913 else if (VecVT == MVT::v2f64)
3914 return PPC::XVCMPGEDP;
3915 break;
3916 default:
3917 break;
3918 }
3919 llvm_unreachable("Invalid floating-point vector compare condition")::llvm::llvm_unreachable_internal("Invalid floating-point vector compare condition"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3919)
;
3920 } else {
3921 /* Handle some cases by swapping input operands. */
3922 switch (CC) {
3923 case ISD::SETGE: CC = ISD::SETLE; Swap = true; break;
3924 case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
3925 case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
3926 case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break;
3927 default: break;
3928 }
3929 /* Handle some cases by negating the result. */
3930 switch (CC) {
3931 case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
3932 case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break;
3933 case ISD::SETLE: CC = ISD::SETGT; Negate = true; break;
3934 case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break;
3935 default: break;
3936 }
3937 /* We have instructions implementing the remaining cases. */
3938 switch (CC) {
3939 case ISD::SETEQ:
3940 case ISD::SETUEQ:
3941 if (VecVT == MVT::v16i8)
3942 return PPC::VCMPEQUB;
3943 else if (VecVT == MVT::v8i16)
3944 return PPC::VCMPEQUH;
3945 else if (VecVT == MVT::v4i32)
3946 return PPC::VCMPEQUW;
3947 else if (VecVT == MVT::v2i64)
3948 return PPC::VCMPEQUD;
3949 break;
3950 case ISD::SETGT:
3951 if (VecVT == MVT::v16i8)
3952 return PPC::VCMPGTSB;
3953 else if (VecVT == MVT::v8i16)
3954 return PPC::VCMPGTSH;
3955 else if (VecVT == MVT::v4i32)
3956 return PPC::VCMPGTSW;
3957 else if (VecVT == MVT::v2i64)
3958 return PPC::VCMPGTSD;
3959 break;
3960 case ISD::SETUGT:
3961 if (VecVT == MVT::v16i8)
3962 return PPC::VCMPGTUB;
3963 else if (VecVT == MVT::v8i16)
3964 return PPC::VCMPGTUH;
3965 else if (VecVT == MVT::v4i32)
3966 return PPC::VCMPGTUW;
3967 else if (VecVT == MVT::v2i64)
3968 return PPC::VCMPGTUD;
3969 break;
3970 default:
3971 break;
3972 }
3973 llvm_unreachable("Invalid integer vector compare condition")::llvm::llvm_unreachable_internal("Invalid integer vector compare condition"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 3973)
;
3974 }
3975}
3976
3977bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
3978 SDLoc dl(N);
3979 unsigned Imm;
3980 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
3981 EVT PtrVT =
3982 CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
3983 bool isPPC64 = (PtrVT == MVT::i64);
3984
3985 if (!PPCSubTarget->useCRBits() &&
3986 isInt32Immediate(N->getOperand(1), Imm)) {
3987 // We can codegen setcc op, imm very efficiently compared to a brcond.
3988 // Check for those cases here.
3989 // setcc op, 0
3990 if (Imm == 0) {
3991 SDValue Op = N->getOperand(0);
3992 switch (CC) {
3993 default: break;
3994 case ISD::SETEQ: {
3995 Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0);
3996 SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl),
3997 getI32Imm(31, dl) };
3998 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
3999 return true;
4000 }
4001 case ISD::SETNE: {
4002 if (isPPC64) break;
4003 SDValue AD =
4004 SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4005 Op, getI32Imm(~0U, dl)), 0);
4006 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1));
4007 return true;
4008 }
4009 case ISD::SETLT: {
4010 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4011 getI32Imm(31, dl) };
4012 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4013 return true;
4014 }
4015 case ISD::SETGT: {
4016 SDValue T =
4017 SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0);
4018 T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0);
4019 SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl),
4020 getI32Imm(31, dl) };
4021 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4022 return true;
4023 }
4024 }
4025 } else if (Imm == ~0U) { // setcc op, -1
4026 SDValue Op = N->getOperand(0);
4027 switch (CC) {
4028 default: break;
4029 case ISD::SETEQ:
4030 if (isPPC64) break;
4031 Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4032 Op, getI32Imm(1, dl)), 0);
4033 CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32,
4034 SDValue(CurDAG->getMachineNode(PPC::LI, dl,
4035 MVT::i32,
4036 getI32Imm(0, dl)),
4037 0), Op.getValue(1));
4038 return true;
4039 case ISD::SETNE: {
4040 if (isPPC64) break;
4041 Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0);
4042 SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4043 Op, getI32Imm(~0U, dl));
4044 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op,
4045 SDValue(AD, 1));
4046 return true;
4047 }
4048 case ISD::SETLT: {
4049 SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op,
4050 getI32Imm(1, dl)), 0);
4051 SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD,
4052 Op), 0);
4053 SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
4054 getI32Imm(31, dl) };
4055 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4056 return true;
4057 }
4058 case ISD::SETGT: {
4059 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4060 getI32Imm(31, dl) };
4061 Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4062 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl));
4063 return true;
4064 }
4065 }
4066 }
4067 }
4068
4069 SDValue LHS = N->getOperand(0);
4070 SDValue RHS = N->getOperand(1);
4071
4072 // Altivec Vector compare instructions do not set any CR register by default and
4073 // vector compare operations return the same type as the operands.
4074 if (LHS.getValueType().isVector()) {
4075 if (PPCSubTarget->hasQPX() || PPCSubTarget->hasSPE())
4076 return false;
4077
4078 EVT VecVT = LHS.getValueType();
4079 bool Swap, Negate;
4080 unsigned int VCmpInst = getVCmpInst(VecVT.getSimpleVT(), CC,
4081 PPCSubTarget->hasVSX(), Swap, Negate);
4082 if (Swap)
4083 std::swap(LHS, RHS);
4084
4085 EVT ResVT = VecVT.changeVectorElementTypeToInteger();
4086 if (Negate) {
4087 SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0);
4088 CurDAG->SelectNodeTo(N, PPCSubTarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
4089 ResVT, VCmp, VCmp);
4090 return true;
4091 }
4092
4093 CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS);
4094 return true;
4095 }
4096
4097 if (PPCSubTarget->useCRBits())
4098 return false;
4099
4100 bool Inv;
4101 unsigned Idx = getCRIdxForSetCC(CC, Inv);
4102 SDValue CCReg = SelectCC(LHS, RHS, CC, dl);
4103 SDValue IntCR;
4104
4105 // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that
4106 // The correct compare instruction is already set by SelectCC()
4107 if (PPCSubTarget->hasSPE() && LHS.getValueType().isFloatingPoint()) {
4108 Idx = 1;
4109 }
4110
4111 // Force the ccreg into CR7.
4112 SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32);
4113
4114 SDValue InFlag(nullptr, 0); // Null incoming flag value.
4115 CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg,
4116 InFlag).getValue(1);
4117
4118 IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg,
4119 CCReg), 0);
4120
4121 SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl),
4122 getI32Imm(31, dl), getI32Imm(31, dl) };
4123 if (!Inv) {
4124 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4125 return true;
4126 }
4127
4128 // Get the specified bit.
4129 SDValue Tmp =
4130 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4131 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl));
4132 return true;
4133}
4134
4135/// Does this node represent a load/store node whose address can be represented
4136/// with a register plus an immediate that's a multiple of \p Val:
4137bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const {
4138 LoadSDNode *LDN = dyn_cast<LoadSDNode>(N);
4139 StoreSDNode *STN = dyn_cast<StoreSDNode>(N);
4140 SDValue AddrOp;
4141 if (LDN)
4142 AddrOp = LDN->getOperand(1);
4143 else if (STN)
4144 AddrOp = STN->getOperand(2);
4145
4146 // If the address points a frame object or a frame object with an offset,
4147 // we need to check the object alignment.
4148 short Imm = 0;
4149 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(
4150 AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) :
4151 AddrOp)) {
4152 // If op0 is a frame index that is under aligned, we can't do it either,
4153 // because it is translated to r31 or r1 + slot + offset. We won't know the
4154 // slot number until the stack frame is finalized.
4155 const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo();
4156 unsigned SlotAlign = MFI.getObjectAlignment(FI->getIndex());
4157 if ((SlotAlign % Val) != 0)
4158 return false;
4159
4160 // If we have an offset, we need further check on the offset.
4161 if (AddrOp.getOpcode() != ISD::ADD)
4162 return true;
4163 }
4164
4165 if (AddrOp.getOpcode() == ISD::ADD)
4166 return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val);
4167
4168 // If the address comes from the outside, the offset will be zero.
4169 return AddrOp.getOpcode() == ISD::CopyFromReg;
4170}
4171
4172void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
4173 // Transfer memoperands.
4174 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
4175 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
4176}
4177
4178static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG,
4179 bool &NeedSwapOps, bool &IsUnCmp) {
4180
4181 assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here.")((N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here."
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::SELECT_CC && \"Expecting a SELECT_CC here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4181, __PRETTY_FUNCTION__))
;
4182
4183 SDValue LHS = N->getOperand(0);
4184 SDValue RHS = N->getOperand(1);
4185 SDValue TrueRes = N->getOperand(2);
4186 SDValue FalseRes = N->getOperand(3);
4187 ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(TrueRes);
4188 if (!TrueConst)
4189 return false;
4190
4191 assert((N->getSimpleValueType(0) == MVT::i64 ||(((N->getSimpleValueType(0) == MVT::i64 || N->getSimpleValueType
(0) == MVT::i32) && "Expecting either i64 or i32 here."
) ? static_cast<void> (0) : __assert_fail ("(N->getSimpleValueType(0) == MVT::i64 || N->getSimpleValueType(0) == MVT::i32) && \"Expecting either i64 or i32 here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
4192 N->getSimpleValueType(0) == MVT::i32) &&(((N->getSimpleValueType(0) == MVT::i64 || N->getSimpleValueType
(0) == MVT::i32) && "Expecting either i64 or i32 here."
) ? static_cast<void> (0) : __assert_fail ("(N->getSimpleValueType(0) == MVT::i64 || N->getSimpleValueType(0) == MVT::i32) && \"Expecting either i64 or i32 here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
4193 "Expecting either i64 or i32 here.")(((N->getSimpleValueType(0) == MVT::i64 || N->getSimpleValueType
(0) == MVT::i32) && "Expecting either i64 or i32 here."
) ? static_cast<void> (0) : __assert_fail ("(N->getSimpleValueType(0) == MVT::i64 || N->getSimpleValueType(0) == MVT::i32) && \"Expecting either i64 or i32 here.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4193, __PRETTY_FUNCTION__))
;
4194
4195 // We are looking for any of:
4196 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4197 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4198 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq)
4199 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq)
4200 int64_t TrueResVal = TrueConst->getSExtValue();
4201 if ((TrueResVal < -1 || TrueResVal > 1) ||
4202 (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) ||
4203 (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) ||
4204 (TrueResVal == 0 &&
4205 (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ)))
4206 return false;
4207
4208 bool InnerIsSel = FalseRes.getOpcode() == ISD::SELECT_CC;
4209 SDValue SetOrSelCC = InnerIsSel ? FalseRes : FalseRes.getOperand(0);
4210 if (SetOrSelCC.getOpcode() != ISD::SETCC &&
4211 SetOrSelCC.getOpcode() != ISD::SELECT_CC)
4212 return false;
4213
4214 // Without this setb optimization, the outer SELECT_CC will be manually
4215 // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass
4216 // transforms pseudo instruction to isel instruction. When there are more than
4217 // one use for result like zext/sext, with current optimization we only see
4218 // isel is replaced by setb but can't see any significant gain. Since
4219 // setb has longer latency than original isel, we should avoid this. Another
4220 // point is that setb requires comparison always kept, it can break the
4221 // opportunity to get the comparison away if we have in future.
4222 if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse()))
4223 return false;
4224
4225 SDValue InnerLHS = SetOrSelCC.getOperand(0);
4226 SDValue InnerRHS = SetOrSelCC.getOperand(1);
4227 ISD::CondCode InnerCC =
4228 cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get();
4229 // If the inner comparison is a select_cc, make sure the true/false values are
4230 // 1/-1 and canonicalize it if needed.
4231 if (InnerIsSel) {
4232 ConstantSDNode *SelCCTrueConst =
4233 dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(2));
4234 ConstantSDNode *SelCCFalseConst =
4235 dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(3));
4236 if (!SelCCTrueConst || !SelCCFalseConst)
4237 return false;
4238 int64_t SelCCTVal = SelCCTrueConst->getSExtValue();
4239 int64_t SelCCFVal = SelCCFalseConst->getSExtValue();
4240 // The values must be -1/1 (requiring a swap) or 1/-1.
4241 if (SelCCTVal == -1 && SelCCFVal == 1) {
4242 std::swap(InnerLHS, InnerRHS);
4243 } else if (SelCCTVal != 1 || SelCCFVal != -1)
4244 return false;
4245 }
4246
4247 // Canonicalize unsigned case
4248 if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) {
4249 IsUnCmp = true;
4250 InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT;
4251 }
4252
4253 bool InnerSwapped = false;
4254 if (LHS == InnerRHS && RHS == InnerLHS)
4255 InnerSwapped = true;
4256 else if (LHS != InnerLHS || RHS != InnerRHS)
4257 return false;
4258
4259 switch (CC) {
4260 // (select_cc lhs, rhs, 0, \
4261 // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq)
4262 case ISD::SETEQ:
4263 if (!InnerIsSel)
4264 return false;
4265 if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT)
4266 return false;
4267 NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped;
4268 break;
4269
4270 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4271 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt)
4272 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt)
4273 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4274 // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt)
4275 // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt)
4276 case ISD::SETULT:
4277 if (!IsUnCmp && InnerCC != ISD::SETNE)
4278 return false;
4279 IsUnCmp = true;
4280 LLVM_FALLTHROUGH[[clang::fallthrough]];
4281 case ISD::SETLT:
4282 if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) ||
4283 (InnerCC == ISD::SETLT && InnerSwapped))
4284 NeedSwapOps = (TrueResVal == 1);
4285 else
4286 return false;
4287 break;
4288
4289 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4290 // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt)
4291 // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt)
4292 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4293 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt)
4294 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt)
4295 case ISD::SETUGT:
4296 if (!IsUnCmp && InnerCC != ISD::SETNE)
4297 return false;
4298 IsUnCmp = true;
4299 LLVM_FALLTHROUGH[[clang::fallthrough]];
4300 case ISD::SETGT:
4301 if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) ||
4302 (InnerCC == ISD::SETGT && InnerSwapped))
4303 NeedSwapOps = (TrueResVal == -1);
4304 else
4305 return false;
4306 break;
4307
4308 default:
4309 return false;
4310 }
4311
4312 LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Found a node that can be lowered to a SETB: "
; } } while (false)
;
4313 LLVM_DEBUG(N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { N->dump(); } } while (false)
;
4314
4315 return true;
4316}
4317
4318// Select - Convert the specified operand from a target-independent to a
4319// target-specific node if it hasn't already been changed.
4320void PPCDAGToDAGISel::Select(SDNode *N) {
4321 SDLoc dl(N);
4322 if (N->isMachineOpcode()) {
4323 N->setNodeId(-1);
4324 return; // Already selected.
4325 }
4326
4327 // In case any misguided DAG-level optimizations form an ADD with a
4328 // TargetConstant operand, crash here instead of miscompiling (by selecting
4329 // an r+r add instead of some kind of r+i add).
4330 if (N->getOpcode() == ISD::ADD &&
4331 N->getOperand(1).getOpcode() == ISD::TargetConstant)
4332 llvm_unreachable("Invalid ADD with TargetConstant operand")::llvm::llvm_unreachable_internal("Invalid ADD with TargetConstant operand"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4332)
;
4333
4334 // Try matching complex bit permutations before doing anything else.
4335 if (tryBitPermutation(N))
4336 return;
4337
4338 // Try to emit integer compares as GPR-only sequences (i.e. no use of CR).
4339 if (tryIntCompareInGPR(N))
4340 return;
4341
4342 switch (N->getOpcode()) {
4343 default: break;
4344
4345 case ISD::Constant:
4346 if (N->getValueType(0) == MVT::i64) {
4347 ReplaceNode(N, selectI64Imm(CurDAG, N));
4348 return;
4349 }
4350 break;
4351
4352 case ISD::SETCC:
4353 if (trySETCC(N))
4354 return;
4355 break;
4356 // These nodes will be transformed into GETtlsADDR32 node, which
4357 // later becomes BL_TLS __tls_get_addr(sym at tlsgd)@PLT
4358 case PPCISD::ADDI_TLSLD_L_ADDR:
4359 case PPCISD::ADDI_TLSGD_L_ADDR: {
4360 const Module *Mod = MF->getFunction().getParent();
4361 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 ||
4362 !PPCSubTarget->isSecurePlt() || !PPCSubTarget->isTargetELF() ||
4363 Mod->getPICLevel() == PICLevel::SmallPIC)
4364 break;
4365 // Attach global base pointer on GETtlsADDR32 node in order to
4366 // generate secure plt code for TLS symbols.
4367 getGlobalBaseReg();
4368 } break;
4369 case PPCISD::CALL: {
4370 const Module *M = MF->getFunction().getParent();
4371
4372 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 ||
4373 (!TM.isPositionIndependent() || !PPCSubTarget->isSecurePlt()) ||
4374 !PPCSubTarget->isTargetELF() || M->getPICLevel() == PICLevel::SmallPIC)
4375 break;
4376
4377 SDValue Op = N->getOperand(1);
4378
4379 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
4380 if (GA->getTargetFlags() == PPCII::MO_PLT)
4381 getGlobalBaseReg();
4382 }
4383 else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) {
4384 if (ES->getTargetFlags() == PPCII::MO_PLT)
4385 getGlobalBaseReg();
4386 }
4387 }
4388 break;
4389
4390 case PPCISD::GlobalBaseReg:
4391 ReplaceNode(N, getGlobalBaseReg());
4392 return;
4393
4394 case ISD::FrameIndex:
4395 selectFrameIndex(N, N);
4396 return;
4397
4398 case PPCISD::MFOCRF: {
4399 SDValue InFlag = N->getOperand(1);
4400 ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32,
4401 N->getOperand(0), InFlag));
4402 return;
4403 }
4404
4405 case PPCISD::READ_TIME_BASE:
4406 ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32,
4407 MVT::Other, N->getOperand(0)));
4408 return;
4409
4410 case PPCISD::SRA_ADDZE: {
4411 SDValue N0 = N->getOperand(0);
4412 SDValue ShiftAmt =
4413 CurDAG->getTargetConstant(*cast<ConstantSDNode>(N->getOperand(1))->
4414 getConstantIntValue(), dl,
4415 N->getValueType(0));
4416 if (N->getValueType(0) == MVT::i64) {
4417 SDNode *Op =
4418 CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue,
4419 N0, ShiftAmt);
4420 CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0),
4421 SDValue(Op, 1));
4422 return;
4423 } else {
4424 assert(N->getValueType(0) == MVT::i32 &&((N->getValueType(0) == MVT::i32 && "Expecting i64 or i32 in PPCISD::SRA_ADDZE"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && \"Expecting i64 or i32 in PPCISD::SRA_ADDZE\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4425, __PRETTY_FUNCTION__))
4425 "Expecting i64 or i32 in PPCISD::SRA_ADDZE")((N->getValueType(0) == MVT::i32 && "Expecting i64 or i32 in PPCISD::SRA_ADDZE"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && \"Expecting i64 or i32 in PPCISD::SRA_ADDZE\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4425, __PRETTY_FUNCTION__))
;
4426 SDNode *Op =
4427 CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue,
4428 N0, ShiftAmt);
4429 CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0),
4430 SDValue(Op, 1));
4431 return;
4432 }
4433 }
4434
4435 case ISD::STORE: {
4436 // Change TLS initial-exec D-form stores to X-form stores.
4437 StoreSDNode *ST = cast<StoreSDNode>(N);
4438 if (EnableTLSOpt && PPCSubTarget->isELFv2ABI() &&
4439 ST->getAddressingMode() != ISD::PRE_INC)
4440 if (tryTLSXFormStore(ST))
4441 return;
4442 break;
4443 }
4444 case ISD::LOAD: {
4445 // Handle preincrement loads.
4446 LoadSDNode *LD = cast<LoadSDNode>(N);
4447 EVT LoadedVT = LD->getMemoryVT();
4448
4449 // Normal loads are handled by code generated from the .td file.
4450 if (LD->getAddressingMode() != ISD::PRE_INC) {
4451 // Change TLS initial-exec D-form loads to X-form loads.
4452 if (EnableTLSOpt && PPCSubTarget->isELFv2ABI())
4453 if (tryTLSXFormLoad(LD))
4454 return;
4455 break;
4456 }
4457
4458 SDValue Offset = LD->getOffset();
4459 if (Offset.getOpcode() == ISD::TargetConstant ||
4460 Offset.getOpcode() == ISD::TargetGlobalAddress) {
4461
4462 unsigned Opcode;
4463 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD;
4464 if (LD->getValueType(0) != MVT::i64) {
4465 // Handle PPC32 integer and normal FP loads.
4466 assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"
) ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16) && \"Invalid sext update load\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4466, __PRETTY_FUNCTION__))
;
4467 switch (LoadedVT.getSimpleVT().SimpleTy) {
4468 default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4468)
;
4469 case MVT::f64: Opcode = PPC::LFDU; break;
4470 case MVT::f32: Opcode = PPC::LFSU; break;
4471 case MVT::i32: Opcode = PPC::LWZU; break;
4472 case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break;
4473 case MVT::i1:
4474 case MVT::i8: Opcode = PPC::LBZU; break;
4475 }
4476 } else {
4477 assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!")((LD->getValueType(0) == MVT::i64 && "Unknown load result type!"
) ? static_cast<void> (0) : __assert_fail ("LD->getValueType(0) == MVT::i64 && \"Unknown load result type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4477, __PRETTY_FUNCTION__))
;
4478 assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"
) ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16) && \"Invalid sext update load\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4478, __PRETTY_FUNCTION__))
;
4479 switch (LoadedVT.getSimpleVT().SimpleTy) {
4480 default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4480)
;
4481 case MVT::i64: Opcode = PPC::LDU; break;
4482 case MVT::i32: Opcode = PPC::LWZU8; break;
4483 case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break;
4484 case MVT::i1:
4485 case MVT::i8: Opcode = PPC::LBZU8; break;
4486 }
4487 }
4488
4489 SDValue Chain = LD->getChain();
4490 SDValue Base = LD->getBasePtr();
4491 SDValue Ops[] = { Offset, Base, Chain };
4492 SDNode *MN = CurDAG->getMachineNode(
4493 Opcode, dl, LD->getValueType(0),
4494 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops);
4495 transferMemOperands(N, MN);
4496 ReplaceNode(N, MN);
4497 return;
4498 } else {
4499 unsigned Opcode;
4500 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD;
4501 if (LD->getValueType(0) != MVT::i64) {
4502 // Handle PPC32 integer and normal FP loads.
4503 assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"
) ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16) && \"Invalid sext update load\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4503, __PRETTY_FUNCTION__))
;
4504 switch (LoadedVT.getSimpleVT().SimpleTy) {
4505 default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4505)
;
4506 case MVT::v4f64: Opcode = PPC::QVLFDUX; break; // QPX
4507 case MVT::v4f32: Opcode = PPC::QVLFSUX; break; // QPX
4508 case MVT::f64: Opcode = PPC::LFDUX; break;
4509 case MVT::f32: Opcode = PPC::LFSUX; break;
4510 case MVT::i32: Opcode = PPC::LWZUX; break;
4511 case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break;
4512 case MVT::i1:
4513 case MVT::i8: Opcode = PPC::LBZUX; break;
4514 }
4515 } else {
4516 assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!")((LD->getValueType(0) == MVT::i64 && "Unknown load result type!"
) ? static_cast<void> (0) : __assert_fail ("LD->getValueType(0) == MVT::i64 && \"Unknown load result type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4516, __PRETTY_FUNCTION__))
;
4517 assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) &&(((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) &&
"Invalid sext update load") ? static_cast<void> (0) : __assert_fail
("(!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && \"Invalid sext update load\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4518, __PRETTY_FUNCTION__))
4518 "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) &&
"Invalid sext update load") ? static_cast<void> (0) : __assert_fail
("(!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && \"Invalid sext update load\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4518, __PRETTY_FUNCTION__))
;
4519 switch (LoadedVT.getSimpleVT().SimpleTy) {
4520 default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4520)
;
4521 case MVT::i64: Opcode = PPC::LDUX; break;
4522 case MVT::i32: Opcode = isSExt ? PPC::LWAUX : PPC::LWZUX8; break;
4523 case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break;
4524 case MVT::i1:
4525 case MVT::i8: Opcode = PPC::LBZUX8; break;
4526 }
4527 }
4528
4529 SDValue Chain = LD->getChain();
4530 SDValue Base = LD->getBasePtr();
4531 SDValue Ops[] = { Base, Offset, Chain };
4532 SDNode *MN = CurDAG->getMachineNode(
4533 Opcode, dl, LD->getValueType(0),
4534 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops);
4535 transferMemOperands(N, MN);
4536 ReplaceNode(N, MN);
4537 return;
4538 }
4539 }
4540
4541 case ISD::AND: {
4542 unsigned Imm, Imm2, SH, MB, ME;
4543 uint64_t Imm64;
4544
4545 // If this is an and of a value rotated between 0 and 31 bits and then and'd
4546 // with a mask, emit rlwinm
4547 if (isInt32Immediate(N->getOperand(1), Imm) &&
4548 isRotateAndMask(N->getOperand(0).getNode(), Imm, false, SH, MB, ME)) {
4549 SDValue Val = N->getOperand(0).getOperand(0);
4550 SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl),
4551 getI32Imm(ME, dl) };
4552 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4553 return;
4554 }
4555 // If this is just a masked value where the input is not handled above, and
4556 // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm
4557 if (isInt32Immediate(N->getOperand(1), Imm) &&
4558 isRunOfOnes(Imm, MB, ME) &&
4559 N->getOperand(0).getOpcode() != ISD::ROTL) {
4560 SDValue Val = N->getOperand(0);
4561 SDValue Ops[] = { Val, getI32Imm(0, dl), getI32Imm(MB, dl),
4562 getI32Imm(ME, dl) };
4563 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4564 return;
4565 }
4566 // If this is a 64-bit zero-extension mask, emit rldicl.
4567 if (isInt64Immediate(N->getOperand(1).getNode(), Imm64) &&
4568 isMask_64(Imm64)) {
4569 SDValue Val = N->getOperand(0);
4570 MB = 64 - countTrailingOnes(Imm64);
4571 SH = 0;
4572
4573 if (Val.getOpcode() == ISD::ANY_EXTEND) {
4574 auto Op0 = Val.getOperand(0);
4575 if ( Op0.getOpcode() == ISD::SRL &&
4576 isInt32Immediate(Op0.getOperand(1).getNode(), Imm) && Imm <= MB) {
4577
4578 auto ResultType = Val.getNode()->getValueType(0);
4579 auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
4580 ResultType);
4581 SDValue IDVal (ImDef, 0);
4582
4583 Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
4584 ResultType, IDVal, Op0.getOperand(0),
4585 getI32Imm(1, dl)), 0);
4586 SH = 64 - Imm;
4587 }
4588 }
4589
4590 // If the operand is a logical right shift, we can fold it into this
4591 // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb)
4592 // for n <= mb. The right shift is really a left rotate followed by a
4593 // mask, and this mask is a more-restrictive sub-mask of the mask implied
4594 // by the shift.
4595 if (Val.getOpcode() == ISD::SRL &&
4596 isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) {
4597 assert(Imm < 64 && "Illegal shift amount")((Imm < 64 && "Illegal shift amount") ? static_cast
<void> (0) : __assert_fail ("Imm < 64 && \"Illegal shift amount\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4597, __PRETTY_FUNCTION__))
;
4598 Val = Val.getOperand(0);
4599 SH = 64 - Imm;
4600 }
4601
4602 SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) };
4603 CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops);
4604 return;
4605 }
4606 // If this is a negated 64-bit zero-extension mask,
4607 // i.e. the immediate is a sequence of ones from most significant side
4608 // and all zero for reminder, we should use rldicr.
4609 if (isInt64Immediate(N->getOperand(1).getNode(), Imm64) &&
4610 isMask_64(~Imm64)) {
4611 SDValue Val = N->getOperand(0);
4612 MB = 63 - countTrailingOnes(~Imm64);
4613 SH = 0;
4614 SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) };
4615 CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops);
4616 return;
4617 }
4618
4619 // AND X, 0 -> 0, not "rlwinm 32".
4620 if (isInt32Immediate(N->getOperand(1), Imm) && (Imm == 0)) {
4621 ReplaceUses(SDValue(N, 0), N->getOperand(1));
4622 return;
4623 }
4624 // ISD::OR doesn't get all the bitfield insertion fun.
4625 // (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a
4626 // bitfield insert.
4627 if (isInt32Immediate(N->getOperand(1), Imm) &&
4628 N->getOperand(0).getOpcode() == ISD::OR &&
4629 isInt32Immediate(N->getOperand(0).getOperand(1), Imm2)) {
4630 // The idea here is to check whether this is equivalent to:
4631 // (c1 & m) | (x & ~m)
4632 // where m is a run-of-ones mask. The logic here is that, for each bit in
4633 // c1 and c2:
4634 // - if both are 1, then the output will be 1.
4635 // - if both are 0, then the output will be 0.
4636 // - if the bit in c1 is 0, and the bit in c2 is 1, then the output will
4637 // come from x.
4638 // - if the bit in c1 is 1, and the bit in c2 is 0, then the output will
4639 // be 0.
4640 // If that last condition is never the case, then we can form m from the
4641 // bits that are the same between c1 and c2.
4642 unsigned MB, ME;
4643 if (isRunOfOnes(~(Imm^Imm2), MB, ME) && !(~Imm & Imm2)) {
4644 SDValue Ops[] = { N->getOperand(0).getOperand(0),
4645 N->getOperand(0).getOperand(1),
4646 getI32Imm(0, dl), getI32Imm(MB, dl),
4647 getI32Imm(ME, dl) };
4648 ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
4649 return;
4650 }
4651 }
4652
4653 // Other cases are autogenerated.
4654 break;
4655 }
4656 case ISD::OR: {
4657 if (N->getValueType(0) == MVT::i32)
4658 if (tryBitfieldInsert(N))
4659 return;
4660
4661 int16_t Imm;
4662 if (N->getOperand(0)->getOpcode() == ISD::FrameIndex &&
4663 isIntS16Immediate(N->getOperand(1), Imm)) {
4664 KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0));
4665
4666 // If this is equivalent to an add, then we can fold it with the
4667 // FrameIndex calculation.
4668 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) {
4669 selectFrameIndex(N, N->getOperand(0).getNode(), (int)Imm);
4670 return;
4671 }
4672 }
4673
4674 // OR with a 32-bit immediate can be handled by ori + oris
4675 // without creating an immediate in a GPR.
4676 uint64_t Imm64 = 0;
4677 bool IsPPC64 = PPCSubTarget->isPPC64();
4678 if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) &&
4679 (Imm64 & ~0xFFFFFFFFuLL) == 0) {
4680 // If ImmHi (ImmHi) is zero, only one ori (oris) is generated later.
4681 uint64_t ImmHi = Imm64 >> 16;
4682 uint64_t ImmLo = Imm64 & 0xFFFF;
4683 if (ImmHi != 0 && ImmLo != 0) {
4684 SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64,
4685 N->getOperand(0),
4686 getI16Imm(ImmLo, dl));
4687 SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)};
4688 CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1);
4689 return;
4690 }
4691 }
4692
4693 // Other cases are autogenerated.
4694 break;
4695 }
4696 case ISD::XOR: {
4697 // XOR with a 32-bit immediate can be handled by xori + xoris
4698 // without creating an immediate in a GPR.
4699 uint64_t Imm64 = 0;
4700 bool IsPPC64 = PPCSubTarget->isPPC64();
4701 if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) &&
4702 (Imm64 & ~0xFFFFFFFFuLL) == 0) {
4703 // If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later.
4704 uint64_t ImmHi = Imm64 >> 16;
4705 uint64_t ImmLo = Imm64 & 0xFFFF;
4706 if (ImmHi != 0 && ImmLo != 0) {
4707 SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
4708 N->getOperand(0),
4709 getI16Imm(ImmLo, dl));
4710 SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)};
4711 CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1);
4712 return;
4713 }
4714 }
4715
4716 break;
4717 }
4718 case ISD::ADD: {
4719 int16_t Imm;
4720 if (N->getOperand(0)->getOpcode() == ISD::FrameIndex &&
4721 isIntS16Immediate(N->getOperand(1), Imm)) {
4722 selectFrameIndex(N, N->getOperand(0).getNode(), (int)Imm);
4723 return;
4724 }
4725
4726 break;
4727 }
4728 case ISD::SHL: {
4729 unsigned Imm, SH, MB, ME;
4730 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) &&
4731 isRotateAndMask(N, Imm, true, SH, MB, ME)) {
4732 SDValue Ops[] = { N->getOperand(0).getOperand(0),
4733 getI32Imm(SH, dl), getI32Imm(MB, dl),
4734 getI32Imm(ME, dl) };
4735 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4736 return;
4737 }
4738
4739 // Other cases are autogenerated.
4740 break;
4741 }
4742 case ISD::SRL: {
4743 unsigned Imm, SH, MB, ME;
4744 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) &&
4745 isRotateAndMask(N, Imm, true, SH, MB, ME)) {
4746 SDValue Ops[] = { N->getOperand(0).getOperand(0),
4747 getI32Imm(SH, dl), getI32Imm(MB, dl),
4748 getI32Imm(ME, dl) };
4749 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4750 return;
4751 }
4752
4753 // Other cases are autogenerated.
4754 break;
4755 }
4756 // FIXME: Remove this once the ANDI glue bug is fixed:
4757 case PPCISD::ANDIo_1_EQ_BIT:
4758 case PPCISD::ANDIo_1_GT_BIT: {
4759 if (!ANDIGlueBug)
4760 break;
4761
4762 EVT InVT = N->getOperand(0).getValueType();
4763 assert((InVT == MVT::i64 || InVT == MVT::i32) &&(((InVT == MVT::i64 || InVT == MVT::i32) && "Invalid input type for ANDIo_1_EQ_BIT"
) ? static_cast<void> (0) : __assert_fail ("(InVT == MVT::i64 || InVT == MVT::i32) && \"Invalid input type for ANDIo_1_EQ_BIT\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4764, __PRETTY_FUNCTION__))
4764 "Invalid input type for ANDIo_1_EQ_BIT")(((InVT == MVT::i64 || InVT == MVT::i32) && "Invalid input type for ANDIo_1_EQ_BIT"
) ? static_cast<void> (0) : __assert_fail ("(InVT == MVT::i64 || InVT == MVT::i32) && \"Invalid input type for ANDIo_1_EQ_BIT\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4764, __PRETTY_FUNCTION__))
;
4765
4766 unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDIo8 : PPC::ANDIo;
4767 SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue,
4768 N->getOperand(0),
4769 CurDAG->getTargetConstant(1, dl, InVT)),
4770 0);
4771 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
4772 SDValue SRIdxVal =
4773 CurDAG->getTargetConstant(N->getOpcode() == PPCISD::ANDIo_1_EQ_BIT ?
4774 PPC::sub_eq : PPC::sub_gt, dl, MVT::i32);
4775
4776 CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg,
4777 SRIdxVal, SDValue(AndI.getNode(), 1) /* glue */);
4778 return;
4779 }
4780 case ISD::SELECT_CC: {
4781 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get();
4782 EVT PtrVT =
4783 CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
4784 bool isPPC64 = (PtrVT == MVT::i64);
4785
4786 // If this is a select of i1 operands, we'll pattern match it.
4787 if (PPCSubTarget->useCRBits() &&
4788 N->getOperand(0).getValueType() == MVT::i1)
4789 break;
4790
4791 if (PPCSubTarget->isISA3_0() && PPCSubTarget->isPPC64()) {
4792 bool NeedSwapOps = false;
4793 bool IsUnCmp = false;
4794 if (mayUseP9Setb(N, CC, CurDAG, NeedSwapOps, IsUnCmp)) {
4795 SDValue LHS = N->getOperand(0);
4796 SDValue RHS = N->getOperand(1);
4797 if (NeedSwapOps)
4798 std::swap(LHS, RHS);
4799
4800 // Make use of SelectCC to generate the comparison to set CR bits, for
4801 // equality comparisons having one literal operand, SelectCC probably
4802 // doesn't need to materialize the whole literal and just use xoris to
4803 // check it first, it leads the following comparison result can't
4804 // exactly represent GT/LT relationship. So to avoid this we specify
4805 // SETGT/SETUGT here instead of SETEQ.
4806 SDValue GenCC =
4807 SelectCC(LHS, RHS, IsUnCmp ? ISD::SETUGT : ISD::SETGT, dl);
4808 CurDAG->SelectNodeTo(
4809 N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB,
4810 N->getValueType(0), GenCC);
4811 NumP9Setb++;
4812 return;
4813 }
4814 }
4815
4816 // Handle the setcc cases here. select_cc lhs, 0, 1, 0, cc
4817 if (!isPPC64)
4818 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)))
4819 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N->getOperand(2)))
4820 if (ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N->getOperand(3)))
4821 if (N1C->isNullValue() && N3C->isNullValue() &&
4822 N2C->getZExtValue() == 1ULL && CC == ISD::SETNE &&
4823 // FIXME: Implement this optzn for PPC64.
4824 N->getValueType(0) == MVT::i32) {
4825 SDNode *Tmp =
4826 CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4827 N->getOperand(0), getI32Imm(~0U, dl));
4828 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0),
4829 N->getOperand(0), SDValue(Tmp, 1));
4830 return;
4831 }
4832
4833 SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl);
4834
4835 if (N->getValueType(0) == MVT::i1) {
4836 // An i1 select is: (c & t) | (!c & f).
4837 bool Inv;
4838 unsigned Idx = getCRIdxForSetCC(CC, Inv);
4839
4840 unsigned SRI;
4841 switch (Idx) {
4842 default: llvm_unreachable("Invalid CC index")::llvm::llvm_unreachable_internal("Invalid CC index", "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4842)
;
4843 case 0: SRI = PPC::sub_lt; break;
4844 case 1: SRI = PPC::sub_gt; break;
4845 case 2: SRI = PPC::sub_eq; break;
4846 case 3: SRI = PPC::sub_un; break;
4847 }
4848
4849 SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg);
4850
4851 SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1,
4852 CCBit, CCBit), 0);
4853 SDValue C = Inv ? NotCCBit : CCBit,
4854 NotC = Inv ? CCBit : NotCCBit;
4855
4856 SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1,
4857 C, N->getOperand(2)), 0);
4858 SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1,
4859 NotC, N->getOperand(3)), 0);
4860
4861 CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF);
4862 return;
4863 }
4864
4865 unsigned BROpc = getPredicateForSetCC(CC);
4866
4867 unsigned SelectCCOp;
4868 if (N->getValueType(0) == MVT::i32)
4869 SelectCCOp = PPC::SELECT_CC_I4;
4870 else if (N->getValueType(0) == MVT::i64)
4871 SelectCCOp = PPC::SELECT_CC_I8;
4872 else if (N->getValueType(0) == MVT::f32) {
4873 if (PPCSubTarget->hasP8Vector())
4874 SelectCCOp = PPC::SELECT_CC_VSSRC;
4875 else if (PPCSubTarget->hasSPE())
4876 SelectCCOp = PPC::SELECT_CC_SPE4;
4877 else
4878 SelectCCOp = PPC::SELECT_CC_F4;
4879 } else if (N->getValueType(0) == MVT::f64) {
4880 if (PPCSubTarget->hasVSX())
4881 SelectCCOp = PPC::SELECT_CC_VSFRC;
4882 else if (PPCSubTarget->hasSPE())
4883 SelectCCOp = PPC::SELECT_CC_SPE;
4884 else
4885 SelectCCOp = PPC::SELECT_CC_F8;
4886 } else if (N->getValueType(0) == MVT::f128)
4887 SelectCCOp = PPC::SELECT_CC_F16;
4888 else if (PPCSubTarget->hasSPE())
4889 SelectCCOp = PPC::SELECT_CC_SPE;
4890 else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f64)
4891 SelectCCOp = PPC::SELECT_CC_QFRC;
4892 else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f32)
4893 SelectCCOp = PPC::SELECT_CC_QSRC;
4894 else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4i1)
4895 SelectCCOp = PPC::SELECT_CC_QBRC;
4896 else if (N->getValueType(0) == MVT::v2f64 ||
4897 N->getValueType(0) == MVT::v2i64)
4898 SelectCCOp = PPC::SELECT_CC_VSRC;
4899 else
4900 SelectCCOp = PPC::SELECT_CC_VRRC;
4901
4902 SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3),
4903 getI32Imm(BROpc, dl) };
4904 CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops);
4905 return;
4906 }
4907 case ISD::VECTOR_SHUFFLE:
4908 if (PPCSubTarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 ||
4909 N->getValueType(0) == MVT::v2i64)) {
4910 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
4911
4912 SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1),
4913 Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1);
4914 unsigned DM[2];
4915
4916 for (int i = 0; i < 2; ++i)
4917 if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2)
4918 DM[i] = 0;
4919 else
4920 DM[i] = 1;
4921
4922 if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 &&
4923 Op1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
4924 isa<LoadSDNode>(Op1.getOperand(0))) {
4925 LoadSDNode *LD = cast<LoadSDNode>(Op1.getOperand(0));
4926 SDValue Base, Offset;
4927
4928 if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() &&
4929 (LD->getMemoryVT() == MVT::f64 ||
4930 LD->getMemoryVT() == MVT::i64) &&
4931 SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) {
4932 SDValue Chain = LD->getChain();
4933 SDValue Ops[] = { Base, Offset, Chain };
4934 MachineMemOperand *MemOp = LD->getMemOperand();
4935 SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX,
4936 N->getValueType(0), Ops);
4937 CurDAG->setNodeMemRefs(cast<MachineSDNode>(NewN), {MemOp});
4938 return;
4939 }
4940 }
4941
4942 // For little endian, we must swap the input operands and adjust
4943 // the mask elements (reverse and invert them).
4944 if (PPCSubTarget->isLittleEndian()) {
4945 std::swap(Op1, Op2);
4946 unsigned tmp = DM[0];
4947 DM[0] = 1 - DM[1];
4948 DM[1] = 1 - tmp;
4949 }
4950
4951 SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl,
4952 MVT::i32);
4953 SDValue Ops[] = { Op1, Op2, DMV };
4954 CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops);
4955 return;
4956 }
4957
4958 break;
4959 case PPCISD::BDNZ:
4960 case PPCISD::BDZ: {
4961 bool IsPPC64 = PPCSubTarget->isPPC64();
4962 SDValue Ops[] = { N->getOperand(1), N->getOperand(0) };
4963 CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ
4964 ? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
4965 : (IsPPC64 ? PPC::BDZ8 : PPC::BDZ),
4966 MVT::Other, Ops);
4967 return;
4968 }
4969 case PPCISD::COND_BRANCH: {
4970 // Op #0 is the Chain.
4971 // Op #1 is the PPC::PRED_* number.
4972 // Op #2 is the CR#
4973 // Op #3 is the Dest MBB
4974 // Op #4 is the Flag.
4975 // Prevent PPC::PRED_* from being selected into LI.
4976 unsigned PCC = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
4977 if (EnableBranchHint)
4978 PCC |= getBranchHint(PCC, FuncInfo, N->getOperand(3));
4979
4980 SDValue Pred = getI32Imm(PCC, dl);
4981 SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3),
4982 N->getOperand(0), N->getOperand(4) };
4983 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
4984 return;
4985 }
4986 case ISD::BR_CC: {
4987 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
4988 unsigned PCC = getPredicateForSetCC(CC);
4989
4990 if (N->getOperand(2).getValueType() == MVT::i1) {
4991 unsigned Opc;
4992 bool Swap;
4993 switch (PCC) {
4994 default: llvm_unreachable("Unexpected Boolean-operand predicate")::llvm::llvm_unreachable_internal("Unexpected Boolean-operand predicate"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 4994)
;
4995 case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true; break;
4996 case PPC::PRED_LE: Opc = PPC::CRORC; Swap = true; break;
4997 case PPC::PRED_EQ: Opc = PPC::CREQV; Swap = false; break;
4998 case PPC::PRED_GE: Opc = PPC::CRORC; Swap = false; break;
4999 case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break;
5000 case PPC::PRED_NE: Opc = PPC::CRXOR; Swap = false; break;
5001 }
5002
5003 // A signed comparison of i1 values produces the opposite result to an
5004 // unsigned one if the condition code includes less-than or greater-than.
5005 // This is because 1 is the most negative signed i1 number and the most
5006 // positive unsigned i1 number. The CR-logical operations used for such
5007 // comparisons are non-commutative so for signed comparisons vs. unsigned
5008 // ones, the input operands just need to be swapped.
5009 if (ISD::isSignedIntSetCC(CC))
5010 Swap = !Swap;
5011
5012 SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1,
5013 N->getOperand(Swap ? 3 : 2),
5014 N->getOperand(Swap ? 2 : 3)), 0);
5015 CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4),
5016 N->getOperand(0));
5017 return;
5018 }
5019
5020 if (EnableBranchHint)
5021 PCC |= getBranchHint(PCC, FuncInfo, N->getOperand(4));
5022
5023 SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl);
5024 SDValue Ops[] = { getI32Imm(PCC, dl), CondCode,
5025 N->getOperand(4), N->getOperand(0) };
5026 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
5027 return;
5028 }
5029 case ISD::BRIND: {
5030 // FIXME: Should custom lower this.
5031 SDValue Chain = N->getOperand(0);
5032 SDValue Target = N->getOperand(1);
5033 unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8;
5034 unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8;
5035 Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target,
5036 Chain), 0);
5037 CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain);
5038 return;
5039 }
5040 case PPCISD::TOC_ENTRY: {
5041 assert ((PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()) &&(((PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()
) && "Only supported for 64-bit ABI and 32-bit SVR4")
? static_cast<void> (0) : __assert_fail ("(PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()) && \"Only supported for 64-bit ABI and 32-bit SVR4\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5042, __PRETTY_FUNCTION__))
5042 "Only supported for 64-bit ABI and 32-bit SVR4")(((PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()
) && "Only supported for 64-bit ABI and 32-bit SVR4")
? static_cast<void> (0) : __assert_fail ("(PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()) && \"Only supported for 64-bit ABI and 32-bit SVR4\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5042, __PRETTY_FUNCTION__))
;
5043 if (PPCSubTarget->isSVR4ABI() && !PPCSubTarget->isPPC64()) {
5044 SDValue GA = N->getOperand(0);
5045 SDNode *MN = CurDAG->getMachineNode(PPC::LWZtoc, dl, MVT::i32, GA,
5046 N->getOperand(1));
5047 transferMemOperands(N, MN);
5048 ReplaceNode(N, MN);
5049 return;
5050 }
5051
5052 // For medium and large code model, we generate two instructions as
5053 // described below. Otherwise we allow SelectCodeCommon to handle this,
5054 // selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA.
5055 CodeModel::Model CModel = TM.getCodeModel();
5056 if (CModel != CodeModel::Medium && CModel != CodeModel::Large)
5057 break;
5058
5059 // The first source operand is a TargetGlobalAddress or a TargetJumpTable.
5060 // If it must be toc-referenced according to PPCSubTarget, we generate:
5061 // LDtocL(@sym, ADDIStocHA(%x2, @sym))
5062 // Otherwise we generate:
5063 // ADDItocL(ADDIStocHA(%x2, @sym), @sym)
5064 SDValue GA = N->getOperand(0);
5065 SDValue TOCbase = N->getOperand(1);
5066 SDNode *Tmp = CurDAG->getMachineNode(PPC::ADDIStocHA, dl, MVT::i64,
5067 TOCbase, GA);
5068 if (PPCLowering->isAccessedAsGotIndirect(GA)) {
5069 // If it is access as got-indirect, we need an extra LD to load
5070 // the address.
5071 SDNode *MN = CurDAG->getMachineNode(PPC::LDtocL, dl, MVT::i64, GA,
5072 SDValue(Tmp, 0));
5073 transferMemOperands(N, MN);
5074 ReplaceNode(N, MN);
5075 return;
5076 }
5077
5078 // Build the address relative to the TOC-pointer..
5079 ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64,
5080 SDValue(Tmp, 0), GA));
5081 return;
5082 }
5083 case PPCISD::PPC32_PICGOT:
5084 // Generate a PIC-safe GOT reference.
5085 assert(!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI() &&((!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI
() && "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4"
) ? static_cast<void> (0) : __assert_fail ("!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI() && \"PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5086, __PRETTY_FUNCTION__))
5086 "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4")((!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI
() && "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4"
) ? static_cast<void> (0) : __assert_fail ("!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI() && \"PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5086, __PRETTY_FUNCTION__))
;
5087 CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT,
5088 PPCLowering->getPointerTy(CurDAG->getDataLayout()),
5089 MVT::i32);
5090 return;
5091
5092 case PPCISD::VADD_SPLAT: {
5093 // This expands into one of three sequences, depending on whether
5094 // the first operand is odd or even, positive or negative.
5095 assert(isa<ConstantSDNode>(N->getOperand(0)) &&((isa<ConstantSDNode>(N->getOperand(0)) && isa
<ConstantSDNode>(N->getOperand(1)) && "Invalid operand on VADD_SPLAT!"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(0)) && isa<ConstantSDNode>(N->getOperand(1)) && \"Invalid operand on VADD_SPLAT!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5097, __PRETTY_FUNCTION__))
5096 isa<ConstantSDNode>(N->getOperand(1)) &&((isa<ConstantSDNode>(N->getOperand(0)) && isa
<ConstantSDNode>(N->getOperand(1)) && "Invalid operand on VADD_SPLAT!"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(0)) && isa<ConstantSDNode>(N->getOperand(1)) && \"Invalid operand on VADD_SPLAT!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5097, __PRETTY_FUNCTION__))
5097 "Invalid operand on VADD_SPLAT!")((isa<ConstantSDNode>(N->getOperand(0)) && isa
<ConstantSDNode>(N->getOperand(1)) && "Invalid operand on VADD_SPLAT!"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(0)) && isa<ConstantSDNode>(N->getOperand(1)) && \"Invalid operand on VADD_SPLAT!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5097, __PRETTY_FUNCTION__))
;
5098
5099 int Elt = N->getConstantOperandVal(0);
5100 int EltSize = N->getConstantOperandVal(1);
5101 unsigned Opc1, Opc2, Opc3;
5102 EVT VT;
5103
5104 if (EltSize == 1) {
5105 Opc1 = PPC::VSPLTISB;
5106 Opc2 = PPC::VADDUBM;
5107 Opc3 = PPC::VSUBUBM;
5108 VT = MVT::v16i8;
5109 } else if (EltSize == 2) {
5110 Opc1 = PPC::VSPLTISH;
5111 Opc2 = PPC::VADDUHM;
5112 Opc3 = PPC::VSUBUHM;
5113 VT = MVT::v8i16;
5114 } else {
5115 assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!")((EltSize == 4 && "Invalid element size on VADD_SPLAT!"
) ? static_cast<void> (0) : __assert_fail ("EltSize == 4 && \"Invalid element size on VADD_SPLAT!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5115, __PRETTY_FUNCTION__))
;
5116 Opc1 = PPC::VSPLTISW;
5117 Opc2 = PPC::VADDUWM;
5118 Opc3 = PPC::VSUBUWM;
5119 VT = MVT::v4i32;
5120 }
5121
5122 if ((Elt & 1) == 0) {
5123 // Elt is even, in the range [-32,-18] + [16,30].
5124 //
5125 // Convert: VADD_SPLAT elt, size
5126 // Into: tmp = VSPLTIS[BHW] elt
5127 // VADDU[BHW]M tmp, tmp
5128 // Where: [BHW] = B for size = 1, H for size = 2, W for size = 4
5129 SDValue EltVal = getI32Imm(Elt >> 1, dl);
5130 SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5131 SDValue TmpVal = SDValue(Tmp, 0);
5132 ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal));
5133 return;
5134 } else if (Elt > 0) {
5135 // Elt is odd and positive, in the range [17,31].
5136 //
5137 // Convert: VADD_SPLAT elt, size
5138 // Into: tmp1 = VSPLTIS[BHW] elt-16
5139 // tmp2 = VSPLTIS[BHW] -16
5140 // VSUBU[BHW]M tmp1, tmp2
5141 SDValue EltVal = getI32Imm(Elt - 16, dl);
5142 SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5143 EltVal = getI32Imm(-16, dl);
5144 SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5145 ReplaceNode(N, CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0),
5146 SDValue(Tmp2, 0)));
5147 return;
5148 } else {
5149 // Elt is odd and negative, in the range [-31,-17].
5150 //
5151 // Convert: VADD_SPLAT elt, size
5152 // Into: tmp1 = VSPLTIS[BHW] elt+16
5153 // tmp2 = VSPLTIS[BHW] -16
5154 // VADDU[BHW]M tmp1, tmp2
5155 SDValue EltVal = getI32Imm(Elt + 16, dl);
5156 SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5157 EltVal = getI32Imm(-16, dl);
5158 SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5159 ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0),
5160 SDValue(Tmp2, 0)));
5161 return;
5162 }
5163 }
5164 }
5165
5166 SelectCode(N);
5167}
5168
5169// If the target supports the cmpb instruction, do the idiom recognition here.
5170// We don't do this as a DAG combine because we don't want to do it as nodes
5171// are being combined (because we might miss part of the eventual idiom). We
5172// don't want to do it during instruction selection because we want to reuse
5173// the logic for lowering the masking operations already part of the
5174// instruction selector.
5175SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) {
5176 SDLoc dl(N);
5177
5178 assert(N->getOpcode() == ISD::OR &&((N->getOpcode() == ISD::OR && "Only OR nodes are supported for CMPB"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Only OR nodes are supported for CMPB\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5179, __PRETTY_FUNCTION__))
5
'?' condition is true
5179 "Only OR nodes are supported for CMPB")((N->getOpcode() == ISD::OR && "Only OR nodes are supported for CMPB"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Only OR nodes are supported for CMPB\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5179, __PRETTY_FUNCTION__))
;
5180
5181 SDValue Res;
5182 if (!PPCSubTarget->hasCMPB())
6
Assuming the condition is false
7
Taking false branch
5183 return Res;
5184
5185 if (N->getValueType(0) != MVT::i32 &&
8
Taking false branch
5186 N->getValueType(0) != MVT::i64)
5187 return Res;
5188
5189 EVT VT = N->getValueType(0);
5190
5191 SDValue RHS, LHS;
5192 bool BytesFound[8] = {false, false, false, false, false, false, false, false};
5193 uint64_t Mask = 0, Alt = 0;
5194
5195 auto IsByteSelectCC = [this](SDValue O, unsigned &b,
5196 uint64_t &Mask, uint64_t &Alt,
5197 SDValue &LHS, SDValue &RHS) {
5198 if (O.getOpcode() != ISD::SELECT_CC)
15
Assuming the condition is false
16
Taking false branch
5199 return false;
5200 ISD::CondCode CC = cast<CondCodeSDNode>(O.getOperand(4))->get();
5201
5202 if (!isa<ConstantSDNode>(O.getOperand(2)) ||
17
Taking false branch
5203 !isa<ConstantSDNode>(O.getOperand(3)))
5204 return false;
5205
5206 uint64_t PM = O.getConstantOperandVal(2);
5207 uint64_t PAlt = O.getConstantOperandVal(3);
5208 for (b = 0; b < 8; ++b) {
18
Loop condition is true. Entering loop body
5209 uint64_t Mask = UINT64_C(0xFF)0xFFUL << (8*b);
5210 if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt)
19
Assuming 'PM' is not equal to 0
20
Assuming the condition is true
21
Assuming the condition is true
22
Taking true branch
5211 break;
23
Execution continues on line 5214
5212 }
5213
5214 if (b == 8)
24
Taking false branch
5215 return false;
5216 Mask |= PM;
5217 Alt |= PAlt;
5218
5219 if (!isa<ConstantSDNode>(O.getOperand(1)) ||
5220 O.getConstantOperandVal(1) != 0) {
5221 SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1);
5222 if (Op0.getOpcode() == ISD::TRUNCATE)
25
Assuming the condition is false
26
Taking false branch
5223 Op0 = Op0.getOperand(0);
5224 if (Op1.getOpcode() == ISD::TRUNCATE)
27
Assuming the condition is false
28
Taking false branch
5225 Op1 = Op1.getOperand(0);
5226
5227 if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL &&
29
Assuming the condition is true
30
Assuming the condition is true
32
Taking true branch
5228 Op0.getOperand(1) == Op1.getOperand(1) && CC == ISD::SETEQ &&
31
Assuming 'CC' is equal to SETEQ
5229 isa<ConstantSDNode>(Op0.getOperand(1))) {
5230
5231 unsigned Bits = Op0.getValueSizeInBits();
5232 if (b != Bits/8-1)
33
Assuming the condition is false
34
Taking false branch
5233 return false;
5234 if (Op0.getConstantOperandVal(1) != Bits-8)
35
Assuming the condition is false
36
Taking false branch
5235 return false;
5236
5237 LHS = Op0.getOperand(0);
5238 RHS = Op1.getOperand(0);
5239 return true;
37
Returning without writing to 'b'
5240 }
5241
5242 // When we have small integers (i16 to be specific), the form present
5243 // post-legalization uses SETULT in the SELECT_CC for the
5244 // higher-order byte, depending on the fact that the
5245 // even-higher-order bytes are known to all be zero, for example:
5246 // select_cc (xor $lhs, $rhs), 256, 65280, 0, setult
5247 // (so when the second byte is the same, because all higher-order
5248 // bits from bytes 3 and 4 are known to be zero, the result of the
5249 // xor can be at most 255)
5250 if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT &&
5251 isa<ConstantSDNode>(O.getOperand(1))) {
5252
5253 uint64_t ULim = O.getConstantOperandVal(1);
5254 if (ULim != (UINT64_C(1)1UL << b*8))
5255 return false;
5256
5257 // Now we need to make sure that the upper bytes are known to be
5258 // zero.
5259 unsigned Bits = Op0.getValueSizeInBits();
5260 if (!CurDAG->MaskedValueIsZero(
5261 Op0, APInt::getHighBitsSet(Bits, Bits - (b + 1) * 8)))
5262 return false;
5263
5264 LHS = Op0.getOperand(0);
5265 RHS = Op0.getOperand(1);
5266 return true;
5267 }
5268
5269 return false;
5270 }
5271
5272 if (CC != ISD::SETEQ)
5273 return false;
5274
5275 SDValue Op = O.getOperand(0);
5276 if (Op.getOpcode() == ISD::AND) {
5277 if (!isa<ConstantSDNode>(Op.getOperand(1)))
5278 return false;
5279 if (Op.getConstantOperandVal(1) != (UINT64_C(0xFF)0xFFUL << (8*b)))
5280 return false;
5281
5282 SDValue XOR = Op.getOperand(0);
5283 if (XOR.getOpcode() == ISD::TRUNCATE)
5284 XOR = XOR.getOperand(0);
5285 if (XOR.getOpcode() != ISD::XOR)
5286 return false;
5287
5288 LHS = XOR.getOperand(0);
5289 RHS = XOR.getOperand(1);
5290 return true;
5291 } else if (Op.getOpcode() == ISD::SRL) {
5292 if (!isa<ConstantSDNode>(Op.getOperand(1)))
5293 return false;
5294 unsigned Bits = Op.getValueSizeInBits();
5295 if (b != Bits/8-1)
5296 return false;
5297 if (Op.getConstantOperandVal(1) != Bits-8)
5298 return false;
5299
5300 SDValue XOR = Op.getOperand(0);
5301 if (XOR.getOpcode() == ISD::TRUNCATE)
5302 XOR = XOR.getOperand(0);
5303 if (XOR.getOpcode() != ISD::XOR)
5304 return false;
5305
5306 LHS = XOR.getOperand(0);
5307 RHS = XOR.getOperand(1);
5308 return true;
5309 }
5310
5311 return false;
5312 };
5313
5314 SmallVector<SDValue, 8> Queue(1, SDValue(N, 0));
5315 while (!Queue.empty()) {
9
Loop condition is true. Entering loop body
5316 SDValue V = Queue.pop_back_val();
5317
5318 for (const SDValue &O : V.getNode()->ops()) {
10
Assuming '__begin2' is not equal to '__end2'
5319 unsigned b;
11
'b' declared without an initial value
5320 uint64_t M = 0, A = 0;
5321 SDValue OLHS, ORHS;
5322 if (O.getOpcode() == ISD::OR) {
12
Assuming the condition is false
13
Taking false branch
5323 Queue.push_back(O);
5324 } else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) {
14
Calling 'operator()'
38
Returning from 'operator()'
39
Taking true branch
5325 if (!LHS) {
40
Taking true branch
5326 LHS = OLHS;
5327 RHS = ORHS;
5328 BytesFound[b] = true;
41
Array subscript is undefined
5329 Mask |= M;
5330 Alt |= A;
5331 } else if ((LHS == ORHS && RHS == OLHS) ||
5332 (RHS == ORHS && LHS == OLHS)) {
5333 BytesFound[b] = true;
5334 Mask |= M;
5335 Alt |= A;
5336 } else {
5337 return Res;
5338 }
5339 } else {
5340 return Res;
5341 }
5342 }
5343 }
5344
5345 unsigned LastB = 0, BCnt = 0;
5346 for (unsigned i = 0; i < 8; ++i)
5347 if (BytesFound[LastB]) {
5348 ++BCnt;
5349 LastB = i;
5350 }
5351
5352 if (!LastB || BCnt < 2)
5353 return Res;
5354
5355 // Because we'll be zero-extending the output anyway if don't have a specific
5356 // value for each input byte (via the Mask), we can 'anyext' the inputs.
5357 if (LHS.getValueType() != VT) {
5358 LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT);
5359 RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT);
5360 }
5361
5362 Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS);
5363
5364 bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1)-1L;
5365 if (NonTrivialMask && !Alt) {
5366 // Res = Mask & CMPB
5367 Res = CurDAG->getNode(ISD::AND, dl, VT, Res,
5368 CurDAG->getConstant(Mask, dl, VT));
5369 } else if (Alt) {
5370 // Res = (CMPB & Mask) | (~CMPB & Alt)
5371 // Which, as suggested here:
5372 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
5373 // can be written as:
5374 // Res = Alt ^ ((Alt ^ Mask) & CMPB)
5375 // useful because the (Alt ^ Mask) can be pre-computed.
5376 Res = CurDAG->getNode(ISD::AND, dl, VT, Res,
5377 CurDAG->getConstant(Mask ^ Alt, dl, VT));
5378 Res = CurDAG->getNode(ISD::XOR, dl, VT, Res,
5379 CurDAG->getConstant(Alt, dl, VT));
5380 }
5381
5382 return Res;
5383}
5384
5385// When CR bit registers are enabled, an extension of an i1 variable to a i32
5386// or i64 value is lowered in terms of a SELECT_I[48] operation, and thus
5387// involves constant materialization of a 0 or a 1 or both. If the result of
5388// the extension is then operated upon by some operator that can be constant
5389// folded with a constant 0 or 1, and that constant can be materialized using
5390// only one instruction (like a zero or one), then we should fold in those
5391// operations with the select.
5392void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) {
5393 if (!PPCSubTarget->useCRBits())
5394 return;
5395
5396 if (N->getOpcode() != ISD::ZERO_EXTEND &&
5397 N->getOpcode() != ISD::SIGN_EXTEND &&
5398 N->getOpcode() != ISD::ANY_EXTEND)
5399 return;
5400
5401 if (N->getOperand(0).getValueType() != MVT::i1)
5402 return;
5403
5404 if (!N->hasOneUse())
5405 return;
5406
5407 SDLoc dl(N);
5408 EVT VT = N->getValueType(0);
5409 SDValue Cond = N->getOperand(0);
5410 SDValue ConstTrue =
5411 CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT);
5412 SDValue ConstFalse = CurDAG->getConstant(0, dl, VT);
5413
5414 do {
5415 SDNode *User = *N->use_begin();
5416 if (User->getNumOperands() != 2)
5417 break;
5418
5419 auto TryFold = [this, N, User, dl](SDValue Val) {
5420 SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1);
5421 SDValue O0 = UserO0.getNode() == N ? Val : UserO0;
5422 SDValue O1 = UserO1.getNode() == N ? Val : UserO1;
5423
5424 return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl,
5425 User->getValueType(0),
5426 O0.getNode(), O1.getNode());
5427 };
5428
5429 // FIXME: When the semantics of the interaction between select and undef
5430 // are clearly defined, it may turn out to be unnecessary to break here.
5431 SDValue TrueRes = TryFold(ConstTrue);
5432 if (!TrueRes || TrueRes.isUndef())
5433 break;
5434 SDValue FalseRes = TryFold(ConstFalse);
5435 if (!FalseRes || FalseRes.isUndef())
5436 break;
5437
5438 // For us to materialize these using one instruction, we must be able to
5439 // represent them as signed 16-bit integers.
5440 uint64_t True = cast<ConstantSDNode>(TrueRes)->getZExtValue(),
5441 False = cast<ConstantSDNode>(FalseRes)->getZExtValue();
5442 if (!isInt<16>(True) || !isInt<16>(False))
5443 break;
5444
5445 // We can replace User with a new SELECT node, and try again to see if we
5446 // can fold the select with its user.
5447 Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes);
5448 N = User;
5449 ConstTrue = TrueRes;
5450 ConstFalse = FalseRes;
5451 } while (N->hasOneUse());
5452}
5453
5454void PPCDAGToDAGISel::PreprocessISelDAG() {
5455 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5456
5457 bool MadeChange = false;
5458 while (Position != CurDAG->allnodes_begin()) {
1
Loop condition is true. Entering loop body
5459 SDNode *N = &*--Position;
5460 if (N->use_empty())
2
Taking false branch
5461 continue;
5462
5463 SDValue Res;
5464 switch (N->getOpcode()) {
3
Control jumps to 'case OR:' at line 5466
5465 default: break;
5466 case ISD::OR:
5467 Res = combineToCMPB(N);
4
Calling 'PPCDAGToDAGISel::combineToCMPB'
5468 break;
5469 }
5470
5471 if (!Res)
5472 foldBoolExts(Res, N);
5473
5474 if (Res) {
5475 LLVM_DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "PPC DAG preprocessing replacing:\nOld: "
; } } while (false)
;
5476 LLVM_DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { N->dump(CurDAG); } } while (false)
;
5477 LLVM_DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\nNew: "; } } while (false
)
;
5478 LLVM_DEBUG(Res.getNode()->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { Res.getNode()->dump(CurDAG); } } while (
false)
;
5479 LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\n"; } } while (false)
;
5480
5481 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
5482 MadeChange = true;
5483 }
5484 }
5485
5486 if (MadeChange)
5487 CurDAG->RemoveDeadNodes();
5488}
5489
5490/// PostprocessISelDAG - Perform some late peephole optimizations
5491/// on the DAG representation.
5492void PPCDAGToDAGISel::PostprocessISelDAG() {
5493 // Skip peepholes at -O0.
5494 if (TM.getOptLevel() == CodeGenOpt::None)
5495 return;
5496
5497 PeepholePPC64();
5498 PeepholeCROps();
5499 PeepholePPC64ZExt();
5500}
5501
5502// Check if all users of this node will become isel where the second operand
5503// is the constant zero. If this is so, and if we can negate the condition,
5504// then we can flip the true and false operands. This will allow the zero to
5505// be folded with the isel so that we don't need to materialize a register
5506// containing zero.
5507bool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) {
5508 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
5509 UI != UE; ++UI) {
5510 SDNode *User = *UI;
5511 if (!User->isMachineOpcode())
5512 return false;
5513 if (User->getMachineOpcode() != PPC::SELECT_I4 &&
5514 User->getMachineOpcode() != PPC::SELECT_I8)
5515 return false;
5516
5517 SDNode *Op2 = User->getOperand(2).getNode();
5518 if (!Op2->isMachineOpcode())
5519 return false;
5520
5521 if (Op2->getMachineOpcode() != PPC::LI &&
5522 Op2->getMachineOpcode() != PPC::LI8)
5523 return false;
5524
5525 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op2->getOperand(0));
5526 if (!C)
5527 return false;
5528
5529 if (!C->isNullValue())
5530 return false;
5531 }
5532
5533 return true;
5534}
5535
5536void PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) {
5537 SmallVector<SDNode *, 4> ToReplace;
5538 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
5539 UI != UE; ++UI) {
5540 SDNode *User = *UI;
5541 assert((User->getMachineOpcode() == PPC::SELECT_I4 ||(((User->getMachineOpcode() == PPC::SELECT_I4 || User->
getMachineOpcode() == PPC::SELECT_I8) && "Must have all select users"
) ? static_cast<void> (0) : __assert_fail ("(User->getMachineOpcode() == PPC::SELECT_I4 || User->getMachineOpcode() == PPC::SELECT_I8) && \"Must have all select users\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5543, __PRETTY_FUNCTION__))
5542 User->getMachineOpcode() == PPC::SELECT_I8) &&(((User->getMachineOpcode() == PPC::SELECT_I4 || User->
getMachineOpcode() == PPC::SELECT_I8) && "Must have all select users"
) ? static_cast<void> (0) : __assert_fail ("(User->getMachineOpcode() == PPC::SELECT_I4 || User->getMachineOpcode() == PPC::SELECT_I8) && \"Must have all select users\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5543, __PRETTY_FUNCTION__))
5543 "Must have all select users")(((User->getMachineOpcode() == PPC::SELECT_I4 || User->
getMachineOpcode() == PPC::SELECT_I8) && "Must have all select users"
) ? static_cast<void> (0) : __assert_fail ("(User->getMachineOpcode() == PPC::SELECT_I4 || User->getMachineOpcode() == PPC::SELECT_I8) && \"Must have all select users\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 5543, __PRETTY_FUNCTION__))
;
5544 ToReplace.push_back(User);
5545 }
5546
5547 for (SmallVector<SDNode *, 4>::iterator UI = ToReplace.begin(),
5548 UE = ToReplace.end(); UI != UE; ++UI) {
5549 SDNode *User = *UI;
5550 SDNode *ResNode =
5551 CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User),
5552 User->getValueType(0), User->getOperand(0),
5553 User->getOperand(2),
5554 User->getOperand(1));
5555
5556 LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "CR Peephole replacing:\nOld: "
; } } while (false)
;
5557 LLVM_DEBUG(User->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { User->dump(CurDAG); } } while (false)
;
5558 LLVM_DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\nNew: "; } } while (false
)
;
5559 LLVM_DEBUG(ResNode->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { ResNode->dump(CurDAG); } } while (false
)
;
5560 LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\n"; } } while (false)
;
5561
5562 ReplaceUses(User, ResNode);
5563 }
5564}
5565
5566void PPCDAGToDAGISel::PeepholeCROps() {
5567 bool IsModified;
5568 do {
5569 IsModified = false;
5570 for (SDNode &Node : CurDAG->allnodes()) {
5571 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
5572 if (!MachineNode || MachineNode->use_empty())
5573 continue;
5574 SDNode *ResNode = MachineNode;
5575
5576 bool Op1Set = false, Op1Unset = false,
5577 Op1Not = false,
5578 Op2Set = false, Op2Unset = false,
5579 Op2Not = false;
5580
5581 unsigned Opcode = MachineNode->getMachineOpcode();
5582 switch (Opcode) {
5583 default: break;
5584 case PPC::CRAND:
5585 case PPC::CRNAND:
5586 case PPC::CROR:
5587 case PPC::CRXOR:
5588 case PPC::CRNOR:
5589 case PPC::CREQV:
5590 case PPC::CRANDC:
5591 case PPC::CRORC: {
5592 SDValue Op = MachineNode->getOperand(1);
5593 if (Op.isMachineOpcode()) {
5594 if (Op.getMachineOpcode() == PPC::CRSET)
5595 Op2Set = true;
5596 else if (Op.getMachineOpcode() == PPC::CRUNSET)
5597 Op2Unset = true;
5598 else if (Op.getMachineOpcode() == PPC::CRNOR &&
5599 Op.getOperand(0) == Op.getOperand(1))
5600 Op2Not = true;
5601 }
5602 LLVM_FALLTHROUGH[[clang::fallthrough]];
5603 }
5604 case PPC::BC:
5605 case PPC::BCn:
5606 case PPC::SELECT_I4:
5607 case PPC::SELECT_I8:
5608 case PPC::SELECT_F4:
5609 case PPC::SELECT_F8:
5610 case PPC::SELECT_QFRC:
5611 case PPC::SELECT_QSRC:
5612 case PPC::SELECT_QBRC:
5613 case PPC::SELECT_SPE:
5614 case PPC::SELECT_SPE4:
5615 case PPC::SELECT_VRRC:
5616 case PPC::SELECT_VSFRC:
5617 case PPC::SELECT_VSSRC:
5618 case PPC::SELECT_VSRC: {
5619 SDValue Op = MachineNode->getOperand(0);
5620 if (Op.isMachineOpcode()) {
5621 if (Op.getMachineOpcode() == PPC::CRSET)
5622 Op1Set = true;
5623 else if (Op.getMachineOpcode() == PPC::CRUNSET)
5624 Op1Unset = true;
5625 else if (Op.getMachineOpcode() == PPC::CRNOR &&
5626 Op.getOperand(0) == Op.getOperand(1))
5627 Op1Not = true;
5628 }
5629 }
5630 break;
5631 }
5632
5633 bool SelectSwap = false;
5634 switch (Opcode) {
5635 default: break;
5636 case PPC::CRAND:
5637 if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
5638 // x & x = x
5639 ResNode = MachineNode->getOperand(0).getNode();
5640 else if (Op1Set)
5641 // 1 & y = y
5642 ResNode = MachineNode->getOperand(1).getNode();
5643 else if (Op2Set)
5644 // x & 1 = x
5645 ResNode = MachineNode->getOperand(0).getNode();
5646 else if (Op1Unset || Op2Unset)
5647 // x & 0 = 0 & y = 0
5648 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
5649 MVT::i1);
5650 else if (Op1Not)
5651 // ~x & y = andc(y, x)
5652 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
5653 MVT::i1, MachineNode->getOperand(1),
5654 MachineNode->getOperand(0).
5655 getOperand(0));
5656 else if (Op2Not)
5657 // x & ~y = andc(x, y)
5658 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
5659 MVT::i1, MachineNode->getOperand(0),
5660 MachineNode->getOperand(1).
5661 getOperand(0));
5662 else if (AllUsersSelectZero(MachineNode)) {
5663 ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode),
5664 MVT::i1, MachineNode->getOperand(0),
5665 MachineNode->getOperand(1));
5666 SelectSwap = true;
5667 }
5668 break;
5669 case PPC::CRNAND:
5670 if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
5671 // nand(x, x) -> nor(x, x)
5672 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5673 MVT::i1, MachineNode->getOperand(0),
5674 MachineNode->getOperand(0));
5675 else if (Op1Set)
5676 // nand(1, y) -> nor(y, y)
5677 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5678 MVT::i1, MachineNode->getOperand(1),
5679 MachineNode->getOperand(1));
5680 else if (Op2Set)
5681 // nand(x, 1) -> nor(x, x)
5682 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5683 MVT::i1, MachineNode->getOperand(0),
5684 MachineNode->getOperand(0));
5685 else if (Op1Unset || Op2Unset)
5686 // nand(x, 0) = nand(0, y) = 1
5687 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
5688 MVT::i1);
5689 else if (Op1Not)
5690 // nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y)
5691 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
5692 MVT::i1, MachineNode->getOperand(0).
5693 getOperand(0),
5694 MachineNode->getOperand(1));
5695 else if (Op2Not)
5696 // nand(x, ~y) = ~x | y = orc(y, x)
5697 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
5698 MVT::i1, MachineNode->getOperand(1).
5699 getOperand(0),
5700 MachineNode->getOperand(0));
5701 else if (AllUsersSelectZero(MachineNode)) {
5702 ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode),
5703 MVT::i1, MachineNode->getOperand(0),
5704 MachineNode->getOperand(1));
5705 SelectSwap = true;
5706 }
5707 break;
5708 case PPC::CROR:
5709 if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
5710 // x | x = x
5711 ResNode = MachineNode->getOperand(0).getNode();
5712 else if (Op1Set || Op2Set)
5713 // x | 1 = 1 | y = 1
5714 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
5715 MVT::i1);
5716 else if (Op1Unset)
5717 // 0 | y = y
5718 ResNode = MachineNode->getOperand(1).getNode();
5719 else if (Op2Unset)
5720 // x | 0 = x
5721 ResNode = MachineNode->getOperand(0).getNode();
5722 else if (Op1Not)
5723 // ~x | y = orc(y, x)
5724 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
5725 MVT::i1, MachineNode->getOperand(1),
5726 MachineNode->getOperand(0).
5727 getOperand(0));
5728 else if (Op2Not)
5729 // x | ~y = orc(x, y)
5730 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
5731 MVT::i1, MachineNode->getOperand(0),
5732 MachineNode->getOperand(1).
5733 getOperand(0));
5734 else if (AllUsersSelectZero(MachineNode)) {
5735 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5736 MVT::i1, MachineNode->getOperand(0),
5737 MachineNode->getOperand(1));
5738 SelectSwap = true;
5739 }
5740 break;
5741 case PPC::CRXOR:
5742 if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
5743 // xor(x, x) = 0
5744 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
5745 MVT::i1);
5746 else if (Op1Set)
5747 // xor(1, y) -> nor(y, y)
5748 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5749 MVT::i1, MachineNode->getOperand(1),
5750 MachineNode->getOperand(1));
5751 else if (Op2Set)
5752 // xor(x, 1) -> nor(x, x)
5753 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5754 MVT::i1, MachineNode->getOperand(0),
5755 MachineNode->getOperand(0));
5756 else if (Op1Unset)
5757 // xor(0, y) = y
5758 ResNode = MachineNode->getOperand(1).getNode();
5759 else if (Op2Unset)
5760 // xor(x, 0) = x
5761 ResNode = MachineNode->getOperand(0).getNode();
5762 else if (Op1Not)
5763 // xor(~x, y) = eqv(x, y)
5764 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode),
5765 MVT::i1, MachineNode->getOperand(0).
5766 getOperand(0),
5767 MachineNode->getOperand(1));
5768 else if (Op2Not)
5769 // xor(x, ~y) = eqv(x, y)
5770 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode),
5771 MVT::i1, MachineNode->getOperand(0),
5772 MachineNode->getOperand(1).
5773 getOperand(0));
5774 else if (AllUsersSelectZero(MachineNode)) {
5775 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode),
5776 MVT::i1, MachineNode->getOperand(0),
5777 MachineNode->getOperand(1));
5778 SelectSwap = true;
5779 }
5780 break;
5781 case PPC::CRNOR:
5782 if (Op1Set || Op2Set)
5783 // nor(1, y) -> 0
5784 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
5785 MVT::i1);
5786 else if (Op1Unset)
5787 // nor(0, y) = ~y -> nor(y, y)
5788 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5789 MVT::i1, MachineNode->getOperand(1),
5790 MachineNode->getOperand(1));
5791 else if (Op2Unset)
5792 // nor(x, 0) = ~x
5793 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5794 MVT::i1, MachineNode->getOperand(0),
5795 MachineNode->getOperand(0));
5796 else if (Op1Not)
5797 // nor(~x, y) = andc(x, y)
5798 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
5799 MVT::i1, MachineNode->getOperand(0).
5800 getOperand(0),
5801 MachineNode->getOperand(1));
5802 else if (Op2Not)
5803 // nor(x, ~y) = andc(y, x)
5804 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
5805 MVT::i1, MachineNode->getOperand(1).
5806 getOperand(0),
5807 MachineNode->getOperand(0));
5808 else if (AllUsersSelectZero(MachineNode)) {
5809 ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode),
5810 MVT::i1, MachineNode->getOperand(0),
5811 MachineNode->getOperand(1));
5812 SelectSwap = true;
5813 }
5814 break;
5815 case PPC::CREQV:
5816 if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
5817 // eqv(x, x) = 1
5818 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
5819 MVT::i1);
5820 else if (Op1Set)
5821 // eqv(1, y) = y
5822 ResNode = MachineNode->getOperand(1).getNode();
5823 else if (Op2Set)
5824 // eqv(x, 1) = x
5825 ResNode = MachineNode->getOperand(0).getNode();
5826 else if (Op1Unset)
5827 // eqv(0, y) = ~y -> nor(y, y)
5828 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5829 MVT::i1, MachineNode->getOperand(1),
5830 MachineNode->getOperand(1));
5831 else if (Op2Unset)
5832 // eqv(x, 0) = ~x
5833 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5834 MVT::i1, MachineNode->getOperand(0),
5835 MachineNode->getOperand(0));
5836 else if (Op1Not)
5837 // eqv(~x, y) = xor(x, y)
5838 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode),
5839 MVT::i1, MachineNode->getOperand(0).
5840 getOperand(0),
5841 MachineNode->getOperand(1));
5842 else if (Op2Not)
5843 // eqv(x, ~y) = xor(x, y)
5844 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode),
5845 MVT::i1, MachineNode->getOperand(0),
5846 MachineNode->getOperand(1).
5847 getOperand(0));
5848 else if (AllUsersSelectZero(MachineNode)) {
5849 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode),
5850 MVT::i1, MachineNode->getOperand(0),
5851 MachineNode->getOperand(1));
5852 SelectSwap = true;
5853 }
5854 break;
5855 case PPC::CRANDC:
5856 if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
5857 // andc(x, x) = 0
5858 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
5859 MVT::i1);
5860 else if (Op1Set)
5861 // andc(1, y) = ~y
5862 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5863 MVT::i1, MachineNode->getOperand(1),
5864 MachineNode->getOperand(1));
5865 else if (Op1Unset || Op2Set)
5866 // andc(0, y) = andc(x, 1) = 0
5867 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
5868 MVT::i1);
5869 else if (Op2Unset)
5870 // andc(x, 0) = x
5871 ResNode = MachineNode->getOperand(0).getNode();
5872 else if (Op1Not)
5873 // andc(~x, y) = ~(x | y) = nor(x, y)
5874 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5875 MVT::i1, MachineNode->getOperand(0).
5876 getOperand(0),
5877 MachineNode->getOperand(1));
5878 else if (Op2Not)
5879 // andc(x, ~y) = x & y
5880 ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode),
5881 MVT::i1, MachineNode->getOperand(0),
5882 MachineNode->getOperand(1).
5883 getOperand(0));
5884 else if (AllUsersSelectZero(MachineNode)) {
5885 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
5886 MVT::i1, MachineNode->getOperand(1),
5887 MachineNode->getOperand(0));
5888 SelectSwap = true;
5889 }
5890 break;
5891 case PPC::CRORC:
5892 if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
5893 // orc(x, x) = 1
5894 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
5895 MVT::i1);
5896 else if (Op1Set || Op2Unset)
5897 // orc(1, y) = orc(x, 0) = 1
5898 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
5899 MVT::i1);
5900 else if (Op2Set)
5901 // orc(x, 1) = x
5902 ResNode = MachineNode->getOperand(0).getNode();
5903 else if (Op1Unset)
5904 // orc(0, y) = ~y
5905 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
5906 MVT::i1, MachineNode->getOperand(1),
5907 MachineNode->getOperand(1));
5908 else if (Op1Not)
5909 // orc(~x, y) = ~(x & y) = nand(x, y)
5910 ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode),
5911 MVT::i1, MachineNode->getOperand(0).
5912 getOperand(0),
5913 MachineNode->getOperand(1));
5914 else if (Op2Not)
5915 // orc(x, ~y) = x | y
5916 ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode),
5917 MVT::i1, MachineNode->getOperand(0),
5918 MachineNode->getOperand(1).
5919 getOperand(0));
5920 else if (AllUsersSelectZero(MachineNode)) {
5921 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
5922 MVT::i1, MachineNode->getOperand(1),
5923 MachineNode->getOperand(0));
5924 SelectSwap = true;
5925 }
5926 break;
5927 case PPC::SELECT_I4:
5928 case PPC::SELECT_I8:
5929 case PPC::SELECT_F4:
5930 case PPC::SELECT_F8:
5931 case PPC::SELECT_QFRC:
5932 case PPC::SELECT_QSRC:
5933 case PPC::SELECT_QBRC:
5934 case PPC::SELECT_SPE:
5935 case PPC::SELECT_SPE4:
5936 case PPC::SELECT_VRRC:
5937 case PPC::SELECT_VSFRC:
5938 case PPC::SELECT_VSSRC:
5939 case PPC::SELECT_VSRC:
5940 if (Op1Set)
5941 ResNode = MachineNode->getOperand(1).getNode();
5942 else if (Op1Unset)
5943 ResNode = MachineNode->getOperand(2).getNode();
5944 else if (Op1Not)
5945 ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(),
5946 SDLoc(MachineNode),
5947 MachineNode->getValueType(0),
5948 MachineNode->getOperand(0).
5949 getOperand(0),
5950 MachineNode->getOperand(2),
5951 MachineNode->getOperand(1));
5952 break;
5953 case PPC::BC:
5954 case PPC::BCn:
5955 if (Op1Not)
5956 ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn :
5957 PPC::BC,
5958 SDLoc(MachineNode),
5959 MVT::Other,
5960 MachineNode->getOperand(0).
5961 getOperand(0),
5962 MachineNode->getOperand(1),
5963 MachineNode->getOperand(2));
5964 // FIXME: Handle Op1Set, Op1Unset here too.
5965 break;
5966 }
5967
5968 // If we're inverting this node because it is used only by selects that
5969 // we'd like to swap, then swap the selects before the node replacement.
5970 if (SelectSwap)
5971 SwapAllSelectUsers(MachineNode);
5972
5973 if (ResNode != MachineNode) {
5974 LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "CR Peephole replacing:\nOld: "
; } } while (false)
;
5975 LLVM_DEBUG(MachineNode->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { MachineNode->dump(CurDAG); } } while (false
)
;
5976 LLVM_DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\nNew: "; } } while (false
)
;
5977 LLVM_DEBUG(ResNode->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { ResNode->dump(CurDAG); } } while (false
)
;
5978 LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\n"; } } while (false)
;
5979
5980 ReplaceUses(MachineNode, ResNode);
5981 IsModified = true;
5982 }
5983 }
5984 if (IsModified)
5985 CurDAG->RemoveDeadNodes();
5986 } while (IsModified);
5987}
5988
5989// Gather the set of 32-bit operations that are known to have their
5990// higher-order 32 bits zero, where ToPromote contains all such operations.
5991static bool PeepholePPC64ZExtGather(SDValue Op32,
5992 SmallPtrSetImpl<SDNode *> &ToPromote) {
5993 if (!Op32.isMachineOpcode())
5994 return false;
5995
5996 // First, check for the "frontier" instructions (those that will clear the
5997 // higher-order 32 bits.
5998
5999 // For RLWINM and RLWNM, we need to make sure that the mask does not wrap
6000 // around. If it does not, then these instructions will clear the
6001 // higher-order bits.
6002 if ((Op32.getMachineOpcode() == PPC::RLWINM ||
6003 Op32.getMachineOpcode() == PPC::RLWNM) &&
6004 Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) {
6005 ToPromote.insert(Op32.getNode());
6006 return true;
6007 }
6008
6009 // SLW and SRW always clear the higher-order bits.
6010 if (Op32.getMachineOpcode() == PPC::SLW ||
6011 Op32.getMachineOpcode() == PPC::SRW) {
6012 ToPromote.insert(Op32.getNode());
6013 return true;
6014 }
6015
6016 // For LI and LIS, we need the immediate to be positive (so that it is not
6017 // sign extended).
6018 if (Op32.getMachineOpcode() == PPC::LI ||
6019 Op32.getMachineOpcode() == PPC::LIS) {
6020 if (!isUInt<15>(Op32.getConstantOperandVal(0)))
6021 return false;
6022
6023 ToPromote.insert(Op32.getNode());
6024 return true;
6025 }
6026
6027 // LHBRX and LWBRX always clear the higher-order bits.
6028 if (Op32.getMachineOpcode() == PPC::LHBRX ||
6029 Op32.getMachineOpcode() == PPC::LWBRX) {
6030 ToPromote.insert(Op32.getNode());
6031 return true;
6032 }
6033
6034 // CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended.
6035 if (Op32.getMachineOpcode() == PPC::CNTLZW ||
6036 Op32.getMachineOpcode() == PPC::CNTTZW) {
6037 ToPromote.insert(Op32.getNode());
6038 return true;
6039 }
6040
6041 // Next, check for those instructions we can look through.
6042
6043 // Assuming the mask does not wrap around, then the higher-order bits are
6044 // taken directly from the first operand.
6045 if (Op32.getMachineOpcode() == PPC::RLWIMI &&
6046 Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) {
6047 SmallPtrSet<SDNode *, 16> ToPromote1;
6048 if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1))
6049 return false;
6050
6051 ToPromote.insert(Op32.getNode());
6052 ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6053 return true;
6054 }
6055
6056 // For OR, the higher-order bits are zero if that is true for both operands.
6057 // For SELECT_I4, the same is true (but the relevant operand numbers are
6058 // shifted by 1).
6059 if (Op32.getMachineOpcode() == PPC::OR ||
6060 Op32.getMachineOpcode() == PPC::SELECT_I4) {
6061 unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0;
6062 SmallPtrSet<SDNode *, 16> ToPromote1;
6063 if (!PeepholePPC64ZExtGather(Op32.getOperand(B+0), ToPromote1))
6064 return false;
6065 if (!PeepholePPC64ZExtGather(Op32.getOperand(B+1), ToPromote1))
6066 return false;
6067
6068 ToPromote.insert(Op32.getNode());
6069 ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6070 return true;
6071 }
6072
6073 // For ORI and ORIS, we need the higher-order bits of the first operand to be
6074 // zero, and also for the constant to be positive (so that it is not sign
6075 // extended).
6076 if (Op32.getMachineOpcode() == PPC::ORI ||
6077 Op32.getMachineOpcode() == PPC::ORIS) {
6078 SmallPtrSet<SDNode *, 16> ToPromote1;
6079 if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1))
6080 return false;
6081 if (!isUInt<15>(Op32.getConstantOperandVal(1)))
6082 return false;
6083
6084 ToPromote.insert(Op32.getNode());
6085 ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6086 return true;
6087 }
6088
6089 // The higher-order bits of AND are zero if that is true for at least one of
6090 // the operands.
6091 if (Op32.getMachineOpcode() == PPC::AND) {
6092 SmallPtrSet<SDNode *, 16> ToPromote1, ToPromote2;
6093 bool Op0OK =
6094 PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1);
6095 bool Op1OK =
6096 PeepholePPC64ZExtGather(Op32.getOperand(1), ToPromote2);
6097 if (!Op0OK && !Op1OK)
6098 return false;
6099
6100 ToPromote.insert(Op32.getNode());
6101
6102 if (Op0OK)
6103 ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6104
6105 if (Op1OK)
6106 ToPromote.insert(ToPromote2.begin(), ToPromote2.end());
6107
6108 return true;
6109 }
6110
6111 // For ANDI and ANDIS, the higher-order bits are zero if either that is true
6112 // of the first operand, or if the second operand is positive (so that it is
6113 // not sign extended).
6114 if (Op32.getMachineOpcode() == PPC::ANDIo ||
6115 Op32.getMachineOpcode() == PPC::ANDISo) {
6116 SmallPtrSet<SDNode *, 16> ToPromote1;
6117 bool Op0OK =
6118 PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1);
6119 bool Op1OK = isUInt<15>(Op32.getConstantOperandVal(1));
6120 if (!Op0OK && !Op1OK)
6121 return false;
6122
6123 ToPromote.insert(Op32.getNode());
6124
6125 if (Op0OK)
6126 ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6127
6128 return true;
6129 }
6130
6131 return false;
6132}
6133
6134void PPCDAGToDAGISel::PeepholePPC64ZExt() {
6135 if (!PPCSubTarget->isPPC64())
6136 return;
6137
6138 // When we zero-extend from i32 to i64, we use a pattern like this:
6139 // def : Pat<(i64 (zext i32:$in)),
6140 // (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32),
6141 // 0, 32)>;
6142 // There are several 32-bit shift/rotate instructions, however, that will
6143 // clear the higher-order bits of their output, rendering the RLDICL
6144 // unnecessary. When that happens, we remove it here, and redefine the
6145 // relevant 32-bit operation to be a 64-bit operation.
6146
6147 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
6148
6149 bool MadeChange = false;
6150 while (Position != CurDAG->allnodes_begin()) {
6151 SDNode *N = &*--Position;
6152 // Skip dead nodes and any non-machine opcodes.
6153 if (N->use_empty() || !N->isMachineOpcode())
6154 continue;
6155
6156 if (N->getMachineOpcode() != PPC::RLDICL)
6157 continue;
6158
6159 if (N->getConstantOperandVal(1) != 0 ||
6160 N->getConstantOperandVal(2) != 32)
6161 continue;
6162
6163 SDValue ISR = N->getOperand(0);
6164 if (!ISR.isMachineOpcode() ||
6165 ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG)
6166 continue;
6167
6168 if (!ISR.hasOneUse())
6169 continue;
6170
6171 if (ISR.getConstantOperandVal(2) != PPC::sub_32)
6172 continue;
6173
6174 SDValue IDef = ISR.getOperand(0);
6175 if (!IDef.isMachineOpcode() ||
6176 IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF)
6177 continue;
6178
6179 // We now know that we're looking at a canonical i32 -> i64 zext. See if we
6180 // can get rid of it.
6181
6182 SDValue Op32 = ISR->getOperand(1);
6183 if (!Op32.isMachineOpcode())
6184 continue;
6185
6186 // There are some 32-bit instructions that always clear the high-order 32
6187 // bits, there are also some instructions (like AND) that we can look
6188 // through.
6189 SmallPtrSet<SDNode *, 16> ToPromote;
6190 if (!PeepholePPC64ZExtGather(Op32, ToPromote))
6191 continue;
6192
6193 // If the ToPromote set contains nodes that have uses outside of the set
6194 // (except for the original INSERT_SUBREG), then abort the transformation.
6195 bool OutsideUse = false;
6196 for (SDNode *PN : ToPromote) {
6197 for (SDNode *UN : PN->uses()) {
6198 if (!ToPromote.count(UN) && UN != ISR.getNode()) {
6199 OutsideUse = true;
6200 break;
6201 }
6202 }
6203
6204 if (OutsideUse)
6205 break;
6206 }
6207 if (OutsideUse)
6208 continue;
6209
6210 MadeChange = true;
6211
6212 // We now know that this zero extension can be removed by promoting to
6213 // nodes in ToPromote to 64-bit operations, where for operations in the
6214 // frontier of the set, we need to insert INSERT_SUBREGs for their
6215 // operands.
6216 for (SDNode *PN : ToPromote) {
6217 unsigned NewOpcode;
6218 switch (PN->getMachineOpcode()) {
6219 default:
6220 llvm_unreachable("Don't know the 64-bit variant of this instruction")::llvm::llvm_unreachable_internal("Don't know the 64-bit variant of this instruction"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Target/PowerPC/PPCISelDAGToDAG.cpp"
, 6220)
;
6221 case PPC::RLWINM: NewOpcode = PPC::RLWINM8; break;
6222 case PPC::RLWNM: NewOpcode = PPC::RLWNM8; break;
6223 case PPC::SLW: NewOpcode = PPC::SLW8; break;
6224 case PPC::SRW: NewOpcode = PPC::SRW8; break;
6225 case PPC::LI: NewOpcode = PPC::LI8; break;
6226 case PPC::LIS: NewOpcode = PPC::LIS8; break;
6227 case PPC::LHBRX: NewOpcode = PPC::LHBRX8; break;
6228 case PPC::LWBRX: NewOpcode = PPC::LWBRX8; break;
6229 case PPC::CNTLZW: NewOpcode = PPC::CNTLZW8; break;
6230 case PPC::CNTTZW: NewOpcode = PPC::CNTTZW8; break;
6231 case PPC::RLWIMI: NewOpcode = PPC::RLWIMI8; break;
6232 case PPC::OR: NewOpcode = PPC::OR8; break;
6233 case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break;
6234 case PPC::ORI: NewOpcode = PPC::ORI8; break;
6235 case PPC::ORIS: NewOpcode = PPC::ORIS8; break;
6236 case PPC::AND: NewOpcode = PPC::AND8; break;
6237 case PPC::ANDIo: NewOpcode = PPC::ANDIo8; break;
6238 case PPC::ANDISo: NewOpcode = PPC::ANDISo8; break;
6239 }
6240
6241 // Note: During the replacement process, the nodes will be in an
6242 // inconsistent state (some instructions will have operands with values
6243 // of the wrong type). Once done, however, everything should be right
6244 // again.
6245
6246 SmallVector<SDValue, 4> Ops;
6247 for (const SDValue &V : PN->ops()) {
6248 if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 &&
6249 !isa<ConstantSDNode>(V)) {
6250 SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) };
6251 SDNode *ReplOp =
6252 CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V),
6253 ISR.getNode()->getVTList(), ReplOpOps);
6254 Ops.push_back(SDValue(ReplOp, 0));
6255 } else {
6256 Ops.push_back(V);
6257 }
6258 }
6259
6260 // Because all to-be-promoted nodes only have users that are other
6261 // promoted nodes (or the original INSERT_SUBREG), we can safely replace
6262 // the i32 result value type with i64.
6263
6264 SmallVector<EVT, 2> NewVTs;
6265 SDVTList VTs = PN->getVTList();
6266 for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i)
6267 if (VTs.VTs[i] == MVT::i32)
6268 NewVTs.push_back(MVT::i64);
6269 else
6270 NewVTs.push_back(VTs.VTs[i]);
6271
6272 LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "PPC64 ZExt Peephole morphing:\nOld: "
; } } while (false)
;
6273 LLVM_DEBUG(PN->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { PN->dump(CurDAG); } } while (false)
;
6274
6275 CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops);
6276
6277 LLVM_DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\nNew: "; } } while (false
)
;
6278 LLVM_DEBUG(PN->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { PN->dump(CurDAG); } } while (false)
;
6279 LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\n"; } } while (false)
;
6280 }
6281
6282 // Now we replace the original zero extend and its associated INSERT_SUBREG
6283 // with the value feeding the INSERT_SUBREG (which has now been promoted to
6284 // return an i64).
6285
6286 LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "PPC64 ZExt Peephole replacing:\nOld: "
; } } while (false)
;
6287 LLVM_DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { N->dump(CurDAG); } } while (false)
;
6288 LLVM_DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\nNew: "; } } while (false
)
;
6289 LLVM_DEBUG(Op32.getNode()->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { Op32.getNode()->dump(CurDAG); } } while
(false)
;
6290 LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\n"; } } while (false)
;
6291
6292 ReplaceUses(N, Op32.getNode());
6293 }
6294
6295 if (MadeChange)
6296 CurDAG->RemoveDeadNodes();
6297}
6298
6299void PPCDAGToDAGISel::PeepholePPC64() {
6300 // These optimizations are currently supported only for 64-bit SVR4.
6301 if (PPCSubTarget->isDarwin() || !PPCSubTarget->isPPC64())
6302 return;
6303
6304 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
6305
6306 while (Position != CurDAG->allnodes_begin()) {
6307 SDNode *N = &*--Position;
6308 // Skip dead nodes and any non-machine opcodes.
6309 if (N->use_empty() || !N->isMachineOpcode())
6310 continue;
6311
6312 unsigned FirstOp;
6313 unsigned StorageOpcode = N->getMachineOpcode();
6314 bool RequiresMod4Offset = false;
6315
6316 switch (StorageOpcode) {
6317 default: continue;
6318
6319 case PPC::LWA:
6320 case PPC::LD:
6321 case PPC::DFLOADf64:
6322 case PPC::DFLOADf32:
6323 RequiresMod4Offset = true;
6324 LLVM_FALLTHROUGH[[clang::fallthrough]];
6325 case PPC::LBZ:
6326 case PPC::LBZ8:
6327 case PPC::LFD:
6328 case PPC::LFS:
6329 case PPC::LHA:
6330 case PPC::LHA8:
6331 case PPC::LHZ:
6332 case PPC::LHZ8:
6333 case PPC::LWZ:
6334 case PPC::LWZ8:
6335 FirstOp = 0;
6336 break;
6337
6338 case PPC::STD:
6339 case PPC::DFSTOREf64:
6340 case PPC::DFSTOREf32:
6341 RequiresMod4Offset = true;
6342 LLVM_FALLTHROUGH[[clang::fallthrough]];
6343 case PPC::STB:
6344 case PPC::STB8:
6345 case PPC::STFD:
6346 case PPC::STFS:
6347 case PPC::STH:
6348 case PPC::STH8:
6349 case PPC::STW:
6350 case PPC::STW8:
6351 FirstOp = 1;
6352 break;
6353 }
6354
6355 // If this is a load or store with a zero offset, or within the alignment,
6356 // we may be able to fold an add-immediate into the memory operation.
6357 // The check against alignment is below, as it can't occur until we check
6358 // the arguments to N
6359 if (!isa<ConstantSDNode>(N->getOperand(FirstOp)))
6360 continue;
6361
6362 SDValue Base = N->getOperand(FirstOp + 1);
6363 if (!Base.isMachineOpcode())
6364 continue;
6365
6366 unsigned Flags = 0;
6367 bool ReplaceFlags = true;
6368
6369 // When the feeding operation is an add-immediate of some sort,
6370 // determine whether we need to add relocation information to the
6371 // target flags on the immediate operand when we fold it into the
6372 // load instruction.
6373 //
6374 // For something like ADDItocL, the relocation information is
6375 // inferred from the opcode; when we process it in the AsmPrinter,
6376 // we add the necessary relocation there. A load, though, can receive
6377 // relocation from various flavors of ADDIxxx, so we need to carry
6378 // the relocation information in the target flags.
6379 switch (Base.getMachineOpcode()) {
6380 default: continue;
6381
6382 case PPC::ADDI8:
6383 case PPC::ADDI:
6384 // In some cases (such as TLS) the relocation information
6385 // is already in place on the operand, so copying the operand
6386 // is sufficient.
6387 ReplaceFlags = false;
6388 // For these cases, the immediate may not be divisible by 4, in
6389 // which case the fold is illegal for DS-form instructions. (The
6390 // other cases provide aligned addresses and are always safe.)
6391 if (RequiresMod4Offset &&
6392 (!isa<ConstantSDNode>(Base.getOperand(1)) ||
6393 Base.getConstantOperandVal(1) % 4 != 0))
6394 continue;
6395 break;
6396 case PPC::ADDIdtprelL:
6397 Flags = PPCII::MO_DTPREL_LO;
6398 break;
6399 case PPC::ADDItlsldL:
6400 Flags = PPCII::MO_TLSLD_LO;
6401 break;
6402 case PPC::ADDItocL:
6403 Flags = PPCII::MO_TOC_LO;
6404 break;
6405 }
6406
6407 SDValue ImmOpnd = Base.getOperand(1);
6408
6409 // On PPC64, the TOC base pointer is guaranteed by the ABI only to have
6410 // 8-byte alignment, and so we can only use offsets less than 8 (otherwise,
6411 // we might have needed different @ha relocation values for the offset
6412 // pointers).
6413 int MaxDisplacement = 7;
6414 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
6415 const GlobalValue *GV = GA->getGlobal();
6416 MaxDisplacement = std::min((int) GV->getAlignment() - 1, MaxDisplacement);
6417 }
6418
6419 bool UpdateHBase = false;
6420 SDValue HBase = Base.getOperand(0);
6421
6422 int Offset = N->getConstantOperandVal(FirstOp);
6423 if (ReplaceFlags) {
6424 if (Offset < 0 || Offset > MaxDisplacement) {
6425 // If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only
6426 // one use, then we can do this for any offset, we just need to also
6427 // update the offset (i.e. the symbol addend) on the addis also.
6428 if (Base.getMachineOpcode() != PPC::ADDItocL)
6429 continue;
6430
6431 if (!HBase.isMachineOpcode() ||
6432 HBase.getMachineOpcode() != PPC::ADDIStocHA)
6433 continue;
6434
6435 if (!Base.hasOneUse() || !HBase.hasOneUse())
6436 continue;
6437
6438 SDValue HImmOpnd = HBase.getOperand(1);
6439 if (HImmOpnd != ImmOpnd)
6440 continue;
6441
6442 UpdateHBase = true;
6443 }
6444 } else {
6445 // If we're directly folding the addend from an addi instruction, then:
6446 // 1. In general, the offset on the memory access must be zero.
6447 // 2. If the addend is a constant, then it can be combined with a
6448 // non-zero offset, but only if the result meets the encoding
6449 // requirements.
6450 if (auto *C = dyn_cast<ConstantSDNode>(ImmOpnd)) {
6451 Offset += C->getSExtValue();
6452
6453 if (RequiresMod4Offset && (Offset % 4) != 0)
6454 continue;
6455
6456 if (!isInt<16>(Offset))
6457 continue;
6458
6459 ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd),
6460 ImmOpnd.getValueType());
6461 } else if (Offset != 0) {
6462 continue;
6463 }
6464 }
6465
6466 // We found an opportunity. Reverse the operands from the add
6467 // immediate and substitute them into the load or store. If
6468 // needed, update the target flags for the immediate operand to
6469 // reflect the necessary relocation information.
6470 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Folding add-immediate into mem-op:\nBase: "
; } } while (false)
;
6471 LLVM_DEBUG(Base->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { Base->dump(CurDAG); } } while (false)
;
6472 LLVM_DEBUG(dbgs() << "\nN: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\nN: "; } } while (false)
;
6473 LLVM_DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { N->dump(CurDAG); } } while (false)
;
6474 LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "\n"; } } while (false)
;
6475
6476 // If the relocation information isn't already present on the
6477 // immediate operand, add it now.
6478 if (ReplaceFlags) {
6479 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
6480 SDLoc dl(GA);
6481 const GlobalValue *GV = GA->getGlobal();
6482 // We can't perform this optimization for data whose alignment
6483 // is insufficient for the instruction encoding.
6484 if (GV->getAlignment() < 4 &&
6485 (RequiresMod4Offset || (Offset % 4) != 0)) {
6486 LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("ppc-codegen")) { dbgs() << "Rejected this candidate for alignment.\n\n"
; } } while (false)
;
6487 continue;
6488 }
6489 ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags);
6490 } else if (ConstantPoolSDNode *CP =
6491 dyn_cast<ConstantPoolSDNode>(ImmOpnd)) {
6492 const Constant *C = CP->getConstVal();
6493 ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64,
6494 CP->getAlignment(),
6495 Offset, Flags);
6496 }
6497 }
6498
6499 if (FirstOp == 1) // Store
6500 (void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd,
6501 Base.getOperand(0), N->getOperand(3));
6502 else // Load
6503 (void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0),
6504 N->getOperand(2));
6505
6506 if (UpdateHBase)
6507 (void)CurDAG->UpdateNodeOperands(HBase.getNode(), HBase.getOperand(0),
6508 ImmOpnd);
6509
6510 // The add-immediate may now be dead, in which case remove it.
6511 if (Base.getNode()->use_empty())
6512 CurDAG->RemoveDeadNode(Base.getNode());
6513 }
6514}
6515
6516/// createPPCISelDag - This pass converts a legalized DAG into a
6517/// PowerPC-specific DAG, ready for instruction scheduling.
6518///
6519FunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM,
6520 CodeGenOpt::Level OptLevel) {
6521 return new PPCDAGToDAGISel(TM, OptLevel);
6522}