File: | lib/Target/PowerPC/PPCISelDAGToDAG.cpp |
Location: | line 1944, column 5 |
Description: | Function call argument is an uninitialized value |
1 | //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===// | |||||
2 | // | |||||
3 | // The LLVM Compiler Infrastructure | |||||
4 | // | |||||
5 | // This file is distributed under the University of Illinois Open Source | |||||
6 | // License. See LICENSE.TXT for details. | |||||
7 | // | |||||
8 | //===----------------------------------------------------------------------===// | |||||
9 | // | |||||
10 | // This file defines a pattern matching instruction selector for PowerPC, | |||||
11 | // converting from a legalized dag to a PPC dag. | |||||
12 | // | |||||
13 | //===----------------------------------------------------------------------===// | |||||
14 | ||||||
15 | #include "PPC.h" | |||||
16 | #include "MCTargetDesc/PPCPredicates.h" | |||||
17 | #include "PPCMachineFunctionInfo.h" | |||||
18 | #include "PPCTargetMachine.h" | |||||
19 | #include "llvm/Analysis/BranchProbabilityInfo.h" | |||||
20 | #include "llvm/CodeGen/FunctionLoweringInfo.h" | |||||
21 | #include "llvm/CodeGen/MachineFunction.h" | |||||
22 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||||
23 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||||
24 | #include "llvm/CodeGen/SelectionDAG.h" | |||||
25 | #include "llvm/CodeGen/SelectionDAGISel.h" | |||||
26 | #include "llvm/IR/Constants.h" | |||||
27 | #include "llvm/IR/Function.h" | |||||
28 | #include "llvm/IR/GlobalAlias.h" | |||||
29 | #include "llvm/IR/GlobalValue.h" | |||||
30 | #include "llvm/IR/GlobalVariable.h" | |||||
31 | #include "llvm/IR/Intrinsics.h" | |||||
32 | #include "llvm/IR/Module.h" | |||||
33 | #include "llvm/Support/CommandLine.h" | |||||
34 | #include "llvm/Support/Debug.h" | |||||
35 | #include "llvm/Support/ErrorHandling.h" | |||||
36 | #include "llvm/Support/MathExtras.h" | |||||
37 | #include "llvm/Support/raw_ostream.h" | |||||
38 | #include "llvm/Target/TargetOptions.h" | |||||
39 | using namespace llvm; | |||||
40 | ||||||
41 | #define DEBUG_TYPE"ppc-codegen" "ppc-codegen" | |||||
42 | ||||||
43 | // FIXME: Remove this once the bug has been fixed! | |||||
44 | cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug", | |||||
45 | cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden); | |||||
46 | ||||||
47 | static cl::opt<bool> | |||||
48 | UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true), | |||||
49 | cl::desc("use aggressive ppc isel for bit permutations"), | |||||
50 | cl::Hidden); | |||||
51 | static cl::opt<bool> BPermRewriterNoMasking( | |||||
52 | "ppc-bit-perm-rewriter-stress-rotates", | |||||
53 | cl::desc("stress rotate selection in aggressive ppc isel for " | |||||
54 | "bit permutations"), | |||||
55 | cl::Hidden); | |||||
56 | ||||||
57 | static cl::opt<bool> EnableBranchHint( | |||||
58 | "ppc-use-branch-hint", cl::init(true), | |||||
59 | cl::desc("Enable static hinting of branches on ppc"), | |||||
60 | cl::Hidden); | |||||
61 | ||||||
62 | namespace llvm { | |||||
63 | void initializePPCDAGToDAGISelPass(PassRegistry&); | |||||
64 | } | |||||
65 | ||||||
66 | namespace { | |||||
67 | //===--------------------------------------------------------------------===// | |||||
68 | /// PPCDAGToDAGISel - PPC specific code to select PPC machine | |||||
69 | /// instructions for SelectionDAG operations. | |||||
70 | /// | |||||
71 | class PPCDAGToDAGISel : public SelectionDAGISel { | |||||
72 | const PPCTargetMachine &TM; | |||||
73 | const PPCSubtarget *PPCSubTarget; | |||||
74 | const PPCTargetLowering *PPCLowering; | |||||
75 | unsigned GlobalBaseReg; | |||||
76 | public: | |||||
77 | explicit PPCDAGToDAGISel(PPCTargetMachine &tm) | |||||
78 | : SelectionDAGISel(tm), TM(tm) { | |||||
79 | initializePPCDAGToDAGISelPass(*PassRegistry::getPassRegistry()); | |||||
80 | } | |||||
81 | ||||||
82 | bool runOnMachineFunction(MachineFunction &MF) override { | |||||
83 | // Make sure we re-emit a set of the global base reg if necessary | |||||
84 | GlobalBaseReg = 0; | |||||
85 | PPCSubTarget = &MF.getSubtarget<PPCSubtarget>(); | |||||
86 | PPCLowering = PPCSubTarget->getTargetLowering(); | |||||
87 | SelectionDAGISel::runOnMachineFunction(MF); | |||||
88 | ||||||
89 | if (!PPCSubTarget->isSVR4ABI()) | |||||
90 | InsertVRSaveCode(MF); | |||||
91 | ||||||
92 | return true; | |||||
93 | } | |||||
94 | ||||||
95 | void PreprocessISelDAG() override; | |||||
96 | void PostprocessISelDAG() override; | |||||
97 | ||||||
98 | /// getI32Imm - Return a target constant with the specified value, of type | |||||
99 | /// i32. | |||||
100 | inline SDValue getI32Imm(unsigned Imm, SDLoc dl) { | |||||
101 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); | |||||
102 | } | |||||
103 | ||||||
104 | /// getI64Imm - Return a target constant with the specified value, of type | |||||
105 | /// i64. | |||||
106 | inline SDValue getI64Imm(uint64_t Imm, SDLoc dl) { | |||||
107 | return CurDAG->getTargetConstant(Imm, dl, MVT::i64); | |||||
108 | } | |||||
109 | ||||||
110 | /// getSmallIPtrImm - Return a target constant of pointer type. | |||||
111 | inline SDValue getSmallIPtrImm(unsigned Imm, SDLoc dl) { | |||||
112 | return CurDAG->getTargetConstant( | |||||
113 | Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout())); | |||||
114 | } | |||||
115 | ||||||
116 | /// isRotateAndMask - Returns true if Mask and Shift can be folded into a | |||||
117 | /// rotate and mask opcode and mask operation. | |||||
118 | static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask, | |||||
119 | unsigned &SH, unsigned &MB, unsigned &ME); | |||||
120 | ||||||
121 | /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC | |||||
122 | /// base register. Return the virtual register that holds this value. | |||||
123 | SDNode *getGlobalBaseReg(); | |||||
124 | ||||||
125 | SDNode *getFrameIndex(SDNode *SN, SDNode *N, unsigned Offset = 0); | |||||
126 | ||||||
127 | // Select - Convert the specified operand from a target-independent to a | |||||
128 | // target-specific node if it hasn't already been changed. | |||||
129 | SDNode *Select(SDNode *N) override; | |||||
130 | ||||||
131 | SDNode *SelectBitfieldInsert(SDNode *N); | |||||
132 | SDNode *SelectBitPermutation(SDNode *N); | |||||
133 | ||||||
134 | /// SelectCC - Select a comparison of the specified values with the | |||||
135 | /// specified condition code, returning the CR# of the expression. | |||||
136 | SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDLoc dl); | |||||
137 | ||||||
138 | /// SelectAddrImm - Returns true if the address N can be represented by | |||||
139 | /// a base register plus a signed 16-bit displacement [r+imm]. | |||||
140 | bool SelectAddrImm(SDValue N, SDValue &Disp, | |||||
141 | SDValue &Base) { | |||||
142 | return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, false); | |||||
143 | } | |||||
144 | ||||||
145 | /// SelectAddrImmOffs - Return true if the operand is valid for a preinc | |||||
146 | /// immediate field. Note that the operand at this point is already the | |||||
147 | /// result of a prior SelectAddressRegImm call. | |||||
148 | bool SelectAddrImmOffs(SDValue N, SDValue &Out) const { | |||||
149 | if (N.getOpcode() == ISD::TargetConstant || | |||||
150 | N.getOpcode() == ISD::TargetGlobalAddress) { | |||||
151 | Out = N; | |||||
152 | return true; | |||||
153 | } | |||||
154 | ||||||
155 | return false; | |||||
156 | } | |||||
157 | ||||||
158 | /// SelectAddrIdx - Given the specified addressed, check to see if it can be | |||||
159 | /// represented as an indexed [r+r] operation. Returns false if it can | |||||
160 | /// be represented by [r+imm], which are preferred. | |||||
161 | bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) { | |||||
162 | return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG); | |||||
163 | } | |||||
164 | ||||||
165 | /// SelectAddrIdxOnly - Given the specified addressed, force it to be | |||||
166 | /// represented as an indexed [r+r] operation. | |||||
167 | bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) { | |||||
168 | return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG); | |||||
169 | } | |||||
170 | ||||||
171 | /// SelectAddrImmX4 - Returns true if the address N can be represented by | |||||
172 | /// a base register plus a signed 16-bit displacement that is a multiple of 4. | |||||
173 | /// Suitable for use by STD and friends. | |||||
174 | bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) { | |||||
175 | return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, true); | |||||
176 | } | |||||
177 | ||||||
178 | // Select an address into a single register. | |||||
179 | bool SelectAddr(SDValue N, SDValue &Base) { | |||||
180 | Base = N; | |||||
181 | return true; | |||||
182 | } | |||||
183 | ||||||
184 | /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for | |||||
185 | /// inline asm expressions. It is always correct to compute the value into | |||||
186 | /// a register. The case of adding a (possibly relocatable) constant to a | |||||
187 | /// register can be improved, but it is wrong to substitute Reg+Reg for | |||||
188 | /// Reg in an asm, because the load or store opcode would have to change. | |||||
189 | bool SelectInlineAsmMemoryOperand(const SDValue &Op, | |||||
190 | unsigned ConstraintID, | |||||
191 | std::vector<SDValue> &OutOps) override { | |||||
192 | ||||||
193 | switch(ConstraintID) { | |||||
194 | default: | |||||
195 | errs() << "ConstraintID: " << ConstraintID << "\n"; | |||||
196 | llvm_unreachable("Unexpected asm memory constraint")::llvm::llvm_unreachable_internal("Unexpected asm memory constraint" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 196); | |||||
197 | case InlineAsm::Constraint_es: | |||||
198 | case InlineAsm::Constraint_i: | |||||
199 | case InlineAsm::Constraint_m: | |||||
200 | case InlineAsm::Constraint_o: | |||||
201 | case InlineAsm::Constraint_Q: | |||||
202 | case InlineAsm::Constraint_Z: | |||||
203 | case InlineAsm::Constraint_Zy: | |||||
204 | // We need to make sure that this one operand does not end up in r0 | |||||
205 | // (because we might end up lowering this as 0(%op)). | |||||
206 | const TargetRegisterInfo *TRI = PPCSubTarget->getRegisterInfo(); | |||||
207 | const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1); | |||||
208 | SDLoc dl(Op); | |||||
209 | SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); | |||||
210 | SDValue NewOp = | |||||
211 | SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, | |||||
212 | dl, Op.getValueType(), | |||||
213 | Op, RC), 0); | |||||
214 | ||||||
215 | OutOps.push_back(NewOp); | |||||
216 | return false; | |||||
217 | } | |||||
218 | return true; | |||||
219 | } | |||||
220 | ||||||
221 | void InsertVRSaveCode(MachineFunction &MF); | |||||
222 | ||||||
223 | const char *getPassName() const override { | |||||
224 | return "PowerPC DAG->DAG Pattern Instruction Selection"; | |||||
225 | } | |||||
226 | ||||||
227 | // Include the pieces autogenerated from the target description. | |||||
228 | #include "PPCGenDAGISel.inc" | |||||
229 | ||||||
230 | private: | |||||
231 | SDNode *SelectSETCC(SDNode *N); | |||||
232 | ||||||
233 | void PeepholePPC64(); | |||||
234 | void PeepholePPC64ZExt(); | |||||
235 | void PeepholeCROps(); | |||||
236 | ||||||
237 | SDValue combineToCMPB(SDNode *N); | |||||
238 | void foldBoolExts(SDValue &Res, SDNode *&N); | |||||
239 | ||||||
240 | bool AllUsersSelectZero(SDNode *N); | |||||
241 | void SwapAllSelectUsers(SDNode *N); | |||||
242 | ||||||
243 | SDNode *transferMemOperands(SDNode *N, SDNode *Result); | |||||
244 | }; | |||||
245 | } | |||||
246 | ||||||
247 | /// InsertVRSaveCode - Once the entire function has been instruction selected, | |||||
248 | /// all virtual registers are created and all machine instructions are built, | |||||
249 | /// check to see if we need to save/restore VRSAVE. If so, do it. | |||||
250 | void PPCDAGToDAGISel::InsertVRSaveCode(MachineFunction &Fn) { | |||||
251 | // Check to see if this function uses vector registers, which means we have to | |||||
252 | // save and restore the VRSAVE register and update it with the regs we use. | |||||
253 | // | |||||
254 | // In this case, there will be virtual registers of vector type created | |||||
255 | // by the scheduler. Detect them now. | |||||
256 | bool HasVectorVReg = false; | |||||
257 | for (unsigned i = 0, e = RegInfo->getNumVirtRegs(); i != e; ++i) { | |||||
258 | unsigned Reg = TargetRegisterInfo::index2VirtReg(i); | |||||
259 | if (RegInfo->getRegClass(Reg) == &PPC::VRRCRegClass) { | |||||
260 | HasVectorVReg = true; | |||||
261 | break; | |||||
262 | } | |||||
263 | } | |||||
264 | if (!HasVectorVReg) return; // nothing to do. | |||||
265 | ||||||
266 | // If we have a vector register, we want to emit code into the entry and exit | |||||
267 | // blocks to save and restore the VRSAVE register. We do this here (instead | |||||
268 | // of marking all vector instructions as clobbering VRSAVE) for two reasons: | |||||
269 | // | |||||
270 | // 1. This (trivially) reduces the load on the register allocator, by not | |||||
271 | // having to represent the live range of the VRSAVE register. | |||||
272 | // 2. This (more significantly) allows us to create a temporary virtual | |||||
273 | // register to hold the saved VRSAVE value, allowing this temporary to be | |||||
274 | // register allocated, instead of forcing it to be spilled to the stack. | |||||
275 | ||||||
276 | // Create two vregs - one to hold the VRSAVE register that is live-in to the | |||||
277 | // function and one for the value after having bits or'd into it. | |||||
278 | unsigned InVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); | |||||
279 | unsigned UpdatedVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); | |||||
280 | ||||||
281 | const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo(); | |||||
282 | MachineBasicBlock &EntryBB = *Fn.begin(); | |||||
283 | DebugLoc dl; | |||||
284 | // Emit the following code into the entry block: | |||||
285 | // InVRSAVE = MFVRSAVE | |||||
286 | // UpdatedVRSAVE = UPDATE_VRSAVE InVRSAVE | |||||
287 | // MTVRSAVE UpdatedVRSAVE | |||||
288 | MachineBasicBlock::iterator IP = EntryBB.begin(); // Insert Point | |||||
289 | BuildMI(EntryBB, IP, dl, TII.get(PPC::MFVRSAVE), InVRSAVE); | |||||
290 | BuildMI(EntryBB, IP, dl, TII.get(PPC::UPDATE_VRSAVE), | |||||
291 | UpdatedVRSAVE).addReg(InVRSAVE); | |||||
292 | BuildMI(EntryBB, IP, dl, TII.get(PPC::MTVRSAVE)).addReg(UpdatedVRSAVE); | |||||
293 | ||||||
294 | // Find all return blocks, outputting a restore in each epilog. | |||||
295 | for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { | |||||
296 | if (BB->isReturnBlock()) { | |||||
297 | IP = BB->end(); --IP; | |||||
298 | ||||||
299 | // Skip over all terminator instructions, which are part of the return | |||||
300 | // sequence. | |||||
301 | MachineBasicBlock::iterator I2 = IP; | |||||
302 | while (I2 != BB->begin() && (--I2)->isTerminator()) | |||||
303 | IP = I2; | |||||
304 | ||||||
305 | // Emit: MTVRSAVE InVRSave | |||||
306 | BuildMI(*BB, IP, dl, TII.get(PPC::MTVRSAVE)).addReg(InVRSAVE); | |||||
307 | } | |||||
308 | } | |||||
309 | } | |||||
310 | ||||||
311 | ||||||
312 | /// getGlobalBaseReg - Output the instructions required to put the | |||||
313 | /// base address to use for accessing globals into a register. | |||||
314 | /// | |||||
315 | SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { | |||||
316 | if (!GlobalBaseReg) { | |||||
317 | const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo(); | |||||
318 | // Insert the set of GlobalBaseReg into the first MBB of the function | |||||
319 | MachineBasicBlock &FirstMBB = MF->front(); | |||||
320 | MachineBasicBlock::iterator MBBI = FirstMBB.begin(); | |||||
321 | const Module *M = MF->getFunction()->getParent(); | |||||
322 | DebugLoc dl; | |||||
323 | ||||||
324 | if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) { | |||||
325 | if (PPCSubTarget->isTargetELF()) { | |||||
326 | GlobalBaseReg = PPC::R30; | |||||
327 | if (M->getPICLevel() == PICLevel::Small) { | |||||
328 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR)); | |||||
329 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); | |||||
330 | MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); | |||||
331 | } else { | |||||
332 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); | |||||
333 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); | |||||
334 | unsigned TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); | |||||
335 | BuildMI(FirstMBB, MBBI, dl, | |||||
336 | TII.get(PPC::UpdateGBR), GlobalBaseReg) | |||||
337 | .addReg(TempReg, RegState::Define).addReg(GlobalBaseReg); | |||||
338 | MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); | |||||
339 | } | |||||
340 | } else { | |||||
341 | GlobalBaseReg = | |||||
342 | RegInfo->createVirtualRegister(&PPC::GPRC_NOR0RegClass); | |||||
343 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); | |||||
344 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); | |||||
345 | } | |||||
346 | } else { | |||||
347 | GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_NOX0RegClass); | |||||
348 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8)); | |||||
349 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg); | |||||
350 | } | |||||
351 | } | |||||
352 | return CurDAG->getRegister(GlobalBaseReg, | |||||
353 | PPCLowering->getPointerTy(CurDAG->getDataLayout())) | |||||
354 | .getNode(); | |||||
355 | } | |||||
356 | ||||||
357 | /// isIntS16Immediate - This method tests to see if the node is either a 32-bit | |||||
358 | /// or 64-bit immediate, and if the value can be accurately represented as a | |||||
359 | /// sign extension from a 16-bit value. If so, this returns true and the | |||||
360 | /// immediate. | |||||
361 | static bool isIntS16Immediate(SDNode *N, short &Imm) { | |||||
362 | if (N->getOpcode() != ISD::Constant) | |||||
363 | return false; | |||||
364 | ||||||
365 | Imm = (short)cast<ConstantSDNode>(N)->getZExtValue(); | |||||
366 | if (N->getValueType(0) == MVT::i32) | |||||
367 | return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue(); | |||||
368 | else | |||||
369 | return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); | |||||
370 | } | |||||
371 | ||||||
372 | static bool isIntS16Immediate(SDValue Op, short &Imm) { | |||||
373 | return isIntS16Immediate(Op.getNode(), Imm); | |||||
374 | } | |||||
375 | ||||||
376 | ||||||
377 | /// isInt32Immediate - This method tests to see if the node is a 32-bit constant | |||||
378 | /// operand. If so Imm will receive the 32-bit value. | |||||
379 | static bool isInt32Immediate(SDNode *N, unsigned &Imm) { | |||||
380 | if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { | |||||
381 | Imm = cast<ConstantSDNode>(N)->getZExtValue(); | |||||
382 | return true; | |||||
383 | } | |||||
384 | return false; | |||||
385 | } | |||||
386 | ||||||
387 | /// isInt64Immediate - This method tests to see if the node is a 64-bit constant | |||||
388 | /// operand. If so Imm will receive the 64-bit value. | |||||
389 | static bool isInt64Immediate(SDNode *N, uint64_t &Imm) { | |||||
390 | if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) { | |||||
391 | Imm = cast<ConstantSDNode>(N)->getZExtValue(); | |||||
392 | return true; | |||||
393 | } | |||||
394 | return false; | |||||
395 | } | |||||
396 | ||||||
397 | // isInt32Immediate - This method tests to see if a constant operand. | |||||
398 | // If so Imm will receive the 32 bit value. | |||||
399 | static bool isInt32Immediate(SDValue N, unsigned &Imm) { | |||||
400 | return isInt32Immediate(N.getNode(), Imm); | |||||
401 | } | |||||
402 | ||||||
403 | static unsigned getBranchHint(unsigned PCC, FunctionLoweringInfo *FuncInfo, | |||||
404 | const SDValue &DestMBB) { | |||||
405 | assert(isa<BasicBlockSDNode>(DestMBB))((isa<BasicBlockSDNode>(DestMBB)) ? static_cast<void > (0) : __assert_fail ("isa<BasicBlockSDNode>(DestMBB)" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 405, __PRETTY_FUNCTION__)); | |||||
406 | ||||||
407 | if (!FuncInfo->BPI) return PPC::BR_NO_HINT; | |||||
408 | ||||||
409 | const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); | |||||
410 | const TerminatorInst *BBTerm = BB->getTerminator(); | |||||
411 | ||||||
412 | if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT; | |||||
413 | ||||||
414 | const BasicBlock *TBB = BBTerm->getSuccessor(0); | |||||
415 | const BasicBlock *FBB = BBTerm->getSuccessor(1); | |||||
416 | ||||||
417 | auto TProb = FuncInfo->BPI->getEdgeProbability(BB, TBB); | |||||
418 | auto FProb = FuncInfo->BPI->getEdgeProbability(BB, FBB); | |||||
419 | ||||||
420 | // We only want to handle cases which are easy to predict at static time, e.g. | |||||
421 | // C++ throw statement, that is very likely not taken, or calling never | |||||
422 | // returned function, e.g. stdlib exit(). So we set Threshold to filter | |||||
423 | // unwanted cases. | |||||
424 | // | |||||
425 | // Below is LLVM branch weight table, we only want to handle case 1, 2 | |||||
426 | // | |||||
427 | // Case Taken:Nontaken Example | |||||
428 | // 1. Unreachable 1048575:1 C++ throw, stdlib exit(), | |||||
429 | // 2. Invoke-terminating 1:1048575 | |||||
430 | // 3. Coldblock 4:64 __builtin_expect | |||||
431 | // 4. Loop Branch 124:4 For loop | |||||
432 | // 5. PH/ZH/FPH 20:12 | |||||
433 | const uint32_t Threshold = 10000; | |||||
434 | ||||||
435 | if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb)) | |||||
436 | return PPC::BR_NO_HINT; | |||||
437 | ||||||
438 | DEBUG(dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName() << "::"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName() << "::" << BB-> getName() << "'\n" << " -> " << TBB-> getName() << ": " << TProb << "\n" << " -> " << FBB->getName() << ": " << FProb << "\n"; } } while (0) | |||||
439 | << BB->getName() << "'\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName() << "::" << BB-> getName() << "'\n" << " -> " << TBB-> getName() << ": " << TProb << "\n" << " -> " << FBB->getName() << ": " << FProb << "\n"; } } while (0) | |||||
440 | << " -> " << TBB->getName() << ": " << TProb << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName() << "::" << BB-> getName() << "'\n" << " -> " << TBB-> getName() << ": " << TProb << "\n" << " -> " << FBB->getName() << ": " << FProb << "\n"; } } while (0) | |||||
441 | << " -> " << FBB->getName() << ": " << FProb << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName() << "::" << BB-> getName() << "'\n" << " -> " << TBB-> getName() << ": " << TProb << "\n" << " -> " << FBB->getName() << ": " << FProb << "\n"; } } while (0); | |||||
442 | ||||||
443 | const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB); | |||||
444 | ||||||
445 | // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, | |||||
446 | // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock | |||||
447 | if (BBDN->getBasicBlock()->getBasicBlock() != TBB) | |||||
448 | std::swap(TProb, FProb); | |||||
449 | ||||||
450 | return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; | |||||
451 | } | |||||
452 | ||||||
453 | // isOpcWithIntImmediate - This method tests to see if the node is a specific | |||||
454 | // opcode and that it has a immediate integer right operand. | |||||
455 | // If so Imm will receive the 32 bit value. | |||||
456 | static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { | |||||
457 | return N->getOpcode() == Opc | |||||
458 | && isInt32Immediate(N->getOperand(1).getNode(), Imm); | |||||
459 | } | |||||
460 | ||||||
461 | SDNode *PPCDAGToDAGISel::getFrameIndex(SDNode *SN, SDNode *N, unsigned Offset) { | |||||
462 | SDLoc dl(SN); | |||||
463 | int FI = cast<FrameIndexSDNode>(N)->getIndex(); | |||||
464 | SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0)); | |||||
465 | unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8; | |||||
466 | if (SN->hasOneUse()) | |||||
467 | return CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI, | |||||
468 | getSmallIPtrImm(Offset, dl)); | |||||
469 | return CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI, | |||||
470 | getSmallIPtrImm(Offset, dl)); | |||||
471 | } | |||||
472 | ||||||
473 | bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask, | |||||
474 | bool isShiftMask, unsigned &SH, | |||||
475 | unsigned &MB, unsigned &ME) { | |||||
476 | // Don't even go down this path for i64, since different logic will be | |||||
477 | // necessary for rldicl/rldicr/rldimi. | |||||
478 | if (N->getValueType(0) != MVT::i32) | |||||
479 | return false; | |||||
480 | ||||||
481 | unsigned Shift = 32; | |||||
482 | unsigned Indeterminant = ~0; // bit mask marking indeterminant results | |||||
483 | unsigned Opcode = N->getOpcode(); | |||||
484 | if (N->getNumOperands() != 2 || | |||||
485 | !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31)) | |||||
486 | return false; | |||||
487 | ||||||
488 | if (Opcode == ISD::SHL) { | |||||
489 | // apply shift left to mask if it comes first | |||||
490 | if (isShiftMask) Mask = Mask << Shift; | |||||
491 | // determine which bits are made indeterminant by shift | |||||
492 | Indeterminant = ~(0xFFFFFFFFu << Shift); | |||||
493 | } else if (Opcode == ISD::SRL) { | |||||
494 | // apply shift right to mask if it comes first | |||||
495 | if (isShiftMask) Mask = Mask >> Shift; | |||||
496 | // determine which bits are made indeterminant by shift | |||||
497 | Indeterminant = ~(0xFFFFFFFFu >> Shift); | |||||
498 | // adjust for the left rotate | |||||
499 | Shift = 32 - Shift; | |||||
500 | } else if (Opcode == ISD::ROTL) { | |||||
501 | Indeterminant = 0; | |||||
502 | } else { | |||||
503 | return false; | |||||
504 | } | |||||
505 | ||||||
506 | // if the mask doesn't intersect any Indeterminant bits | |||||
507 | if (Mask && !(Mask & Indeterminant)) { | |||||
508 | SH = Shift & 31; | |||||
509 | // make sure the mask is still a mask (wrap arounds may not be) | |||||
510 | return isRunOfOnes(Mask, MB, ME); | |||||
511 | } | |||||
512 | return false; | |||||
513 | } | |||||
514 | ||||||
515 | /// SelectBitfieldInsert - turn an or of two masked values into | |||||
516 | /// the rotate left word immediate then mask insert (rlwimi) instruction. | |||||
517 | SDNode *PPCDAGToDAGISel::SelectBitfieldInsert(SDNode *N) { | |||||
518 | SDValue Op0 = N->getOperand(0); | |||||
519 | SDValue Op1 = N->getOperand(1); | |||||
520 | SDLoc dl(N); | |||||
521 | ||||||
522 | APInt LKZ, LKO, RKZ, RKO; | |||||
523 | CurDAG->computeKnownBits(Op0, LKZ, LKO); | |||||
524 | CurDAG->computeKnownBits(Op1, RKZ, RKO); | |||||
525 | ||||||
526 | unsigned TargetMask = LKZ.getZExtValue(); | |||||
527 | unsigned InsertMask = RKZ.getZExtValue(); | |||||
528 | ||||||
529 | if ((TargetMask | InsertMask) == 0xFFFFFFFF) { | |||||
530 | unsigned Op0Opc = Op0.getOpcode(); | |||||
531 | unsigned Op1Opc = Op1.getOpcode(); | |||||
532 | unsigned Value, SH = 0; | |||||
533 | TargetMask = ~TargetMask; | |||||
534 | InsertMask = ~InsertMask; | |||||
535 | ||||||
536 | // If the LHS has a foldable shift and the RHS does not, then swap it to the | |||||
537 | // RHS so that we can fold the shift into the insert. | |||||
538 | if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) { | |||||
539 | if (Op0.getOperand(0).getOpcode() == ISD::SHL || | |||||
540 | Op0.getOperand(0).getOpcode() == ISD::SRL) { | |||||
541 | if (Op1.getOperand(0).getOpcode() != ISD::SHL && | |||||
542 | Op1.getOperand(0).getOpcode() != ISD::SRL) { | |||||
543 | std::swap(Op0, Op1); | |||||
544 | std::swap(Op0Opc, Op1Opc); | |||||
545 | std::swap(TargetMask, InsertMask); | |||||
546 | } | |||||
547 | } | |||||
548 | } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) { | |||||
549 | if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL && | |||||
550 | Op1.getOperand(0).getOpcode() != ISD::SRL) { | |||||
551 | std::swap(Op0, Op1); | |||||
552 | std::swap(Op0Opc, Op1Opc); | |||||
553 | std::swap(TargetMask, InsertMask); | |||||
554 | } | |||||
555 | } | |||||
556 | ||||||
557 | unsigned MB, ME; | |||||
558 | if (isRunOfOnes(InsertMask, MB, ME)) { | |||||
559 | SDValue Tmp1, Tmp2; | |||||
560 | ||||||
561 | if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) && | |||||
562 | isInt32Immediate(Op1.getOperand(1), Value)) { | |||||
563 | Op1 = Op1.getOperand(0); | |||||
564 | SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value; | |||||
565 | } | |||||
566 | if (Op1Opc == ISD::AND) { | |||||
567 | // The AND mask might not be a constant, and we need to make sure that | |||||
568 | // if we're going to fold the masking with the insert, all bits not | |||||
569 | // know to be zero in the mask are known to be one. | |||||
570 | APInt MKZ, MKO; | |||||
571 | CurDAG->computeKnownBits(Op1.getOperand(1), MKZ, MKO); | |||||
572 | bool CanFoldMask = InsertMask == MKO.getZExtValue(); | |||||
573 | ||||||
574 | unsigned SHOpc = Op1.getOperand(0).getOpcode(); | |||||
575 | if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask && | |||||
576 | isInt32Immediate(Op1.getOperand(0).getOperand(1), Value)) { | |||||
577 | // Note that Value must be in range here (less than 32) because | |||||
578 | // otherwise there would not be any bits set in InsertMask. | |||||
579 | Op1 = Op1.getOperand(0).getOperand(0); | |||||
580 | SH = (SHOpc == ISD::SHL) ? Value : 32 - Value; | |||||
581 | } | |||||
582 | } | |||||
583 | ||||||
584 | SH &= 31; | |||||
585 | SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl), | |||||
586 | getI32Imm(ME, dl) }; | |||||
587 | return CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops); | |||||
588 | } | |||||
589 | } | |||||
590 | return nullptr; | |||||
591 | } | |||||
592 | ||||||
593 | // Predict the number of instructions that would be generated by calling | |||||
594 | // SelectInt64(N). | |||||
595 | static unsigned SelectInt64CountDirect(int64_t Imm) { | |||||
596 | // Assume no remaining bits. | |||||
597 | unsigned Remainder = 0; | |||||
598 | // Assume no shift required. | |||||
599 | unsigned Shift = 0; | |||||
600 | ||||||
601 | // If it can't be represented as a 32 bit value. | |||||
602 | if (!isInt<32>(Imm)) { | |||||
603 | Shift = countTrailingZeros<uint64_t>(Imm); | |||||
604 | int64_t ImmSh = static_cast<uint64_t>(Imm) >> Shift; | |||||
605 | ||||||
606 | // If the shifted value fits 32 bits. | |||||
607 | if (isInt<32>(ImmSh)) { | |||||
608 | // Go with the shifted value. | |||||
609 | Imm = ImmSh; | |||||
610 | } else { | |||||
611 | // Still stuck with a 64 bit value. | |||||
612 | Remainder = Imm; | |||||
613 | Shift = 32; | |||||
614 | Imm >>= 32; | |||||
615 | } | |||||
616 | } | |||||
617 | ||||||
618 | // Intermediate operand. | |||||
619 | unsigned Result = 0; | |||||
620 | ||||||
621 | // Handle first 32 bits. | |||||
622 | unsigned Lo = Imm & 0xFFFF; | |||||
623 | ||||||
624 | // Simple value. | |||||
625 | if (isInt<16>(Imm)) { | |||||
626 | // Just the Lo bits. | |||||
627 | ++Result; | |||||
628 | } else if (Lo) { | |||||
629 | // Handle the Hi bits and Lo bits. | |||||
630 | Result += 2; | |||||
631 | } else { | |||||
632 | // Just the Hi bits. | |||||
633 | ++Result; | |||||
634 | } | |||||
635 | ||||||
636 | // If no shift, we're done. | |||||
637 | if (!Shift) return Result; | |||||
638 | ||||||
639 | // Shift for next step if the upper 32-bits were not zero. | |||||
640 | if (Imm) | |||||
641 | ++Result; | |||||
642 | ||||||
643 | // Add in the last bits as required. | |||||
644 | if ((Remainder >> 16) & 0xFFFF) | |||||
645 | ++Result; | |||||
646 | if (Remainder & 0xFFFF) | |||||
647 | ++Result; | |||||
648 | ||||||
649 | return Result; | |||||
650 | } | |||||
651 | ||||||
652 | static uint64_t Rot64(uint64_t Imm, unsigned R) { | |||||
653 | return (Imm << R) | (Imm >> (64 - R)); | |||||
654 | } | |||||
655 | ||||||
656 | static unsigned SelectInt64Count(int64_t Imm) { | |||||
657 | unsigned Count = SelectInt64CountDirect(Imm); | |||||
658 | if (Count == 1) | |||||
659 | return Count; | |||||
660 | ||||||
661 | for (unsigned r = 1; r < 63; ++r) { | |||||
662 | uint64_t RImm = Rot64(Imm, r); | |||||
663 | unsigned RCount = SelectInt64CountDirect(RImm) + 1; | |||||
664 | Count = std::min(Count, RCount); | |||||
665 | ||||||
666 | // See comments in SelectInt64 for an explanation of the logic below. | |||||
667 | unsigned LS = findLastSet(RImm); | |||||
668 | if (LS != r-1) | |||||
669 | continue; | |||||
670 | ||||||
671 | uint64_t OnesMask = -(int64_t) (UINT64_C(1)1UL << (LS+1)); | |||||
672 | uint64_t RImmWithOnes = RImm | OnesMask; | |||||
673 | ||||||
674 | RCount = SelectInt64CountDirect(RImmWithOnes) + 1; | |||||
675 | Count = std::min(Count, RCount); | |||||
676 | } | |||||
677 | ||||||
678 | return Count; | |||||
679 | } | |||||
680 | ||||||
681 | // Select a 64-bit constant. For cost-modeling purposes, SelectInt64Count | |||||
682 | // (above) needs to be kept in sync with this function. | |||||
683 | static SDNode *SelectInt64Direct(SelectionDAG *CurDAG, SDLoc dl, int64_t Imm) { | |||||
684 | // Assume no remaining bits. | |||||
685 | unsigned Remainder = 0; | |||||
686 | // Assume no shift required. | |||||
687 | unsigned Shift = 0; | |||||
688 | ||||||
689 | // If it can't be represented as a 32 bit value. | |||||
690 | if (!isInt<32>(Imm)) { | |||||
691 | Shift = countTrailingZeros<uint64_t>(Imm); | |||||
692 | int64_t ImmSh = static_cast<uint64_t>(Imm) >> Shift; | |||||
693 | ||||||
694 | // If the shifted value fits 32 bits. | |||||
695 | if (isInt<32>(ImmSh)) { | |||||
696 | // Go with the shifted value. | |||||
697 | Imm = ImmSh; | |||||
698 | } else { | |||||
699 | // Still stuck with a 64 bit value. | |||||
700 | Remainder = Imm; | |||||
701 | Shift = 32; | |||||
702 | Imm >>= 32; | |||||
703 | } | |||||
704 | } | |||||
705 | ||||||
706 | // Intermediate operand. | |||||
707 | SDNode *Result; | |||||
708 | ||||||
709 | // Handle first 32 bits. | |||||
710 | unsigned Lo = Imm & 0xFFFF; | |||||
711 | unsigned Hi = (Imm >> 16) & 0xFFFF; | |||||
712 | ||||||
713 | auto getI32Imm = [CurDAG, dl](unsigned Imm) { | |||||
714 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); | |||||
715 | }; | |||||
716 | ||||||
717 | // Simple value. | |||||
718 | if (isInt<16>(Imm)) { | |||||
719 | // Just the Lo bits. | |||||
720 | Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, getI32Imm(Lo)); | |||||
721 | } else if (Lo) { | |||||
722 | // Handle the Hi bits. | |||||
723 | unsigned OpC = Hi ? PPC::LIS8 : PPC::LI8; | |||||
724 | Result = CurDAG->getMachineNode(OpC, dl, MVT::i64, getI32Imm(Hi)); | |||||
725 | // And Lo bits. | |||||
726 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, | |||||
727 | SDValue(Result, 0), getI32Imm(Lo)); | |||||
728 | } else { | |||||
729 | // Just the Hi bits. | |||||
730 | Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi)); | |||||
731 | } | |||||
732 | ||||||
733 | // If no shift, we're done. | |||||
734 | if (!Shift) return Result; | |||||
735 | ||||||
736 | // Shift for next step if the upper 32-bits were not zero. | |||||
737 | if (Imm) { | |||||
738 | Result = CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, | |||||
739 | SDValue(Result, 0), | |||||
740 | getI32Imm(Shift), | |||||
741 | getI32Imm(63 - Shift)); | |||||
742 | } | |||||
743 | ||||||
744 | // Add in the last bits as required. | |||||
745 | if ((Hi = (Remainder >> 16) & 0xFFFF)) { | |||||
746 | Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, | |||||
747 | SDValue(Result, 0), getI32Imm(Hi)); | |||||
748 | } | |||||
749 | if ((Lo = Remainder & 0xFFFF)) { | |||||
750 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, | |||||
751 | SDValue(Result, 0), getI32Imm(Lo)); | |||||
752 | } | |||||
753 | ||||||
754 | return Result; | |||||
755 | } | |||||
756 | ||||||
757 | static SDNode *SelectInt64(SelectionDAG *CurDAG, SDLoc dl, int64_t Imm) { | |||||
758 | unsigned Count = SelectInt64CountDirect(Imm); | |||||
759 | if (Count == 1) | |||||
760 | return SelectInt64Direct(CurDAG, dl, Imm); | |||||
761 | ||||||
762 | unsigned RMin = 0; | |||||
763 | ||||||
764 | int64_t MatImm; | |||||
765 | unsigned MaskEnd; | |||||
766 | ||||||
767 | for (unsigned r = 1; r < 63; ++r) { | |||||
768 | uint64_t RImm = Rot64(Imm, r); | |||||
769 | unsigned RCount = SelectInt64CountDirect(RImm) + 1; | |||||
770 | if (RCount < Count) { | |||||
771 | Count = RCount; | |||||
772 | RMin = r; | |||||
773 | MatImm = RImm; | |||||
774 | MaskEnd = 63; | |||||
775 | } | |||||
776 | ||||||
777 | // If the immediate to generate has many trailing zeros, it might be | |||||
778 | // worthwhile to generate a rotated value with too many leading ones | |||||
779 | // (because that's free with li/lis's sign-extension semantics), and then | |||||
780 | // mask them off after rotation. | |||||
781 | ||||||
782 | unsigned LS = findLastSet(RImm); | |||||
783 | // We're adding (63-LS) higher-order ones, and we expect to mask them off | |||||
784 | // after performing the inverse rotation by (64-r). So we need that: | |||||
785 | // 63-LS == 64-r => LS == r-1 | |||||
786 | if (LS != r-1) | |||||
787 | continue; | |||||
788 | ||||||
789 | uint64_t OnesMask = -(int64_t) (UINT64_C(1)1UL << (LS+1)); | |||||
790 | uint64_t RImmWithOnes = RImm | OnesMask; | |||||
791 | ||||||
792 | RCount = SelectInt64CountDirect(RImmWithOnes) + 1; | |||||
793 | if (RCount < Count) { | |||||
794 | Count = RCount; | |||||
795 | RMin = r; | |||||
796 | MatImm = RImmWithOnes; | |||||
797 | MaskEnd = LS; | |||||
798 | } | |||||
799 | } | |||||
800 | ||||||
801 | if (!RMin) | |||||
802 | return SelectInt64Direct(CurDAG, dl, Imm); | |||||
803 | ||||||
804 | auto getI32Imm = [CurDAG, dl](unsigned Imm) { | |||||
805 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); | |||||
806 | }; | |||||
807 | ||||||
808 | SDValue Val = SDValue(SelectInt64Direct(CurDAG, dl, MatImm), 0); | |||||
809 | return CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Val, | |||||
810 | getI32Imm(64 - RMin), getI32Imm(MaskEnd)); | |||||
811 | } | |||||
812 | ||||||
813 | // Select a 64-bit constant. | |||||
814 | static SDNode *SelectInt64(SelectionDAG *CurDAG, SDNode *N) { | |||||
815 | SDLoc dl(N); | |||||
816 | ||||||
817 | // Get 64 bit value. | |||||
818 | int64_t Imm = cast<ConstantSDNode>(N)->getZExtValue(); | |||||
819 | return SelectInt64(CurDAG, dl, Imm); | |||||
820 | } | |||||
821 | ||||||
822 | namespace { | |||||
823 | class BitPermutationSelector { | |||||
824 | struct ValueBit { | |||||
825 | SDValue V; | |||||
826 | ||||||
827 | // The bit number in the value, using a convention where bit 0 is the | |||||
828 | // lowest-order bit. | |||||
829 | unsigned Idx; | |||||
830 | ||||||
831 | enum Kind { | |||||
832 | ConstZero, | |||||
833 | Variable | |||||
834 | } K; | |||||
835 | ||||||
836 | ValueBit(SDValue V, unsigned I, Kind K = Variable) | |||||
837 | : V(V), Idx(I), K(K) {} | |||||
838 | ValueBit(Kind K = Variable) | |||||
839 | : V(SDValue(nullptr, 0)), Idx(UINT32_MAX(4294967295U)), K(K) {} | |||||
840 | ||||||
841 | bool isZero() const { | |||||
842 | return K == ConstZero; | |||||
843 | } | |||||
844 | ||||||
845 | bool hasValue() const { | |||||
846 | return K == Variable; | |||||
847 | } | |||||
848 | ||||||
849 | SDValue getValue() const { | |||||
850 | assert(hasValue() && "Cannot get the value of a constant bit")((hasValue() && "Cannot get the value of a constant bit" ) ? static_cast<void> (0) : __assert_fail ("hasValue() && \"Cannot get the value of a constant bit\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 850, __PRETTY_FUNCTION__)); | |||||
851 | return V; | |||||
852 | } | |||||
853 | ||||||
854 | unsigned getValueBitIndex() const { | |||||
855 | assert(hasValue() && "Cannot get the value bit index of a constant bit")((hasValue() && "Cannot get the value bit index of a constant bit" ) ? static_cast<void> (0) : __assert_fail ("hasValue() && \"Cannot get the value bit index of a constant bit\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 855, __PRETTY_FUNCTION__)); | |||||
856 | return Idx; | |||||
857 | } | |||||
858 | }; | |||||
859 | ||||||
860 | // A bit group has the same underlying value and the same rotate factor. | |||||
861 | struct BitGroup { | |||||
862 | SDValue V; | |||||
863 | unsigned RLAmt; | |||||
864 | unsigned StartIdx, EndIdx; | |||||
865 | ||||||
866 | // This rotation amount assumes that the lower 32 bits of the quantity are | |||||
867 | // replicated in the high 32 bits by the rotation operator (which is done | |||||
868 | // by rlwinm and friends in 64-bit mode). | |||||
869 | bool Repl32; | |||||
870 | // Did converting to Repl32 == true change the rotation factor? If it did, | |||||
871 | // it decreased it by 32. | |||||
872 | bool Repl32CR; | |||||
873 | // Was this group coalesced after setting Repl32 to true? | |||||
874 | bool Repl32Coalesced; | |||||
875 | ||||||
876 | BitGroup(SDValue V, unsigned R, unsigned S, unsigned E) | |||||
877 | : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false), | |||||
878 | Repl32Coalesced(false) { | |||||
879 | DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R << " [" << S << ", " << E << "]\n"; } } while (0) | |||||
880 | " [" << S << ", " << E << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R << " [" << S << ", " << E << "]\n"; } } while (0); | |||||
881 | } | |||||
882 | }; | |||||
883 | ||||||
884 | // Information on each (Value, RLAmt) pair (like the number of groups | |||||
885 | // associated with each) used to choose the lowering method. | |||||
886 | struct ValueRotInfo { | |||||
887 | SDValue V; | |||||
888 | unsigned RLAmt; | |||||
889 | unsigned NumGroups; | |||||
890 | unsigned FirstGroupStartIdx; | |||||
891 | bool Repl32; | |||||
892 | ||||||
893 | ValueRotInfo() | |||||
894 | : RLAmt(UINT32_MAX(4294967295U)), NumGroups(0), FirstGroupStartIdx(UINT32_MAX(4294967295U)), | |||||
895 | Repl32(false) {} | |||||
896 | ||||||
897 | // For sorting (in reverse order) by NumGroups, and then by | |||||
898 | // FirstGroupStartIdx. | |||||
899 | bool operator < (const ValueRotInfo &Other) const { | |||||
900 | // We need to sort so that the non-Repl32 come first because, when we're | |||||
901 | // doing masking, the Repl32 bit groups might be subsumed into the 64-bit | |||||
902 | // masking operation. | |||||
903 | if (Repl32 < Other.Repl32) | |||||
904 | return true; | |||||
905 | else if (Repl32 > Other.Repl32) | |||||
906 | return false; | |||||
907 | else if (NumGroups > Other.NumGroups) | |||||
908 | return true; | |||||
909 | else if (NumGroups < Other.NumGroups) | |||||
910 | return false; | |||||
911 | else if (FirstGroupStartIdx < Other.FirstGroupStartIdx) | |||||
912 | return true; | |||||
913 | return false; | |||||
914 | } | |||||
915 | }; | |||||
916 | ||||||
917 | // Return true if something interesting was deduced, return false if we're | |||||
918 | // providing only a generic representation of V (or something else likewise | |||||
919 | // uninteresting for instruction selection). | |||||
920 | bool getValueBits(SDValue V, SmallVector<ValueBit, 64> &Bits) { | |||||
921 | switch (V.getOpcode()) { | |||||
922 | default: break; | |||||
923 | case ISD::ROTL: | |||||
924 | if (isa<ConstantSDNode>(V.getOperand(1))) { | |||||
925 | unsigned RotAmt = V.getConstantOperandVal(1); | |||||
926 | ||||||
927 | SmallVector<ValueBit, 64> LHSBits(Bits.size()); | |||||
928 | getValueBits(V.getOperand(0), LHSBits); | |||||
929 | ||||||
930 | for (unsigned i = 0; i < Bits.size(); ++i) | |||||
931 | Bits[i] = LHSBits[i < RotAmt ? i + (Bits.size() - RotAmt) : i - RotAmt]; | |||||
932 | ||||||
933 | return true; | |||||
934 | } | |||||
935 | break; | |||||
936 | case ISD::SHL: | |||||
937 | if (isa<ConstantSDNode>(V.getOperand(1))) { | |||||
938 | unsigned ShiftAmt = V.getConstantOperandVal(1); | |||||
939 | ||||||
940 | SmallVector<ValueBit, 64> LHSBits(Bits.size()); | |||||
941 | getValueBits(V.getOperand(0), LHSBits); | |||||
942 | ||||||
943 | for (unsigned i = ShiftAmt; i < Bits.size(); ++i) | |||||
944 | Bits[i] = LHSBits[i - ShiftAmt]; | |||||
945 | ||||||
946 | for (unsigned i = 0; i < ShiftAmt; ++i) | |||||
947 | Bits[i] = ValueBit(ValueBit::ConstZero); | |||||
948 | ||||||
949 | return true; | |||||
950 | } | |||||
951 | break; | |||||
952 | case ISD::SRL: | |||||
953 | if (isa<ConstantSDNode>(V.getOperand(1))) { | |||||
954 | unsigned ShiftAmt = V.getConstantOperandVal(1); | |||||
955 | ||||||
956 | SmallVector<ValueBit, 64> LHSBits(Bits.size()); | |||||
957 | getValueBits(V.getOperand(0), LHSBits); | |||||
958 | ||||||
959 | for (unsigned i = 0; i < Bits.size() - ShiftAmt; ++i) | |||||
960 | Bits[i] = LHSBits[i + ShiftAmt]; | |||||
961 | ||||||
962 | for (unsigned i = Bits.size() - ShiftAmt; i < Bits.size(); ++i) | |||||
963 | Bits[i] = ValueBit(ValueBit::ConstZero); | |||||
964 | ||||||
965 | return true; | |||||
966 | } | |||||
967 | break; | |||||
968 | case ISD::AND: | |||||
969 | if (isa<ConstantSDNode>(V.getOperand(1))) { | |||||
970 | uint64_t Mask = V.getConstantOperandVal(1); | |||||
971 | ||||||
972 | SmallVector<ValueBit, 64> LHSBits(Bits.size()); | |||||
973 | bool LHSTrivial = getValueBits(V.getOperand(0), LHSBits); | |||||
974 | ||||||
975 | for (unsigned i = 0; i < Bits.size(); ++i) | |||||
976 | if (((Mask >> i) & 1) == 1) | |||||
977 | Bits[i] = LHSBits[i]; | |||||
978 | else | |||||
979 | Bits[i] = ValueBit(ValueBit::ConstZero); | |||||
980 | ||||||
981 | // Mark this as interesting, only if the LHS was also interesting. This | |||||
982 | // prevents the overall procedure from matching a single immediate 'and' | |||||
983 | // (which is non-optimal because such an and might be folded with other | |||||
984 | // things if we don't select it here). | |||||
985 | return LHSTrivial; | |||||
986 | } | |||||
987 | break; | |||||
988 | case ISD::OR: { | |||||
989 | SmallVector<ValueBit, 64> LHSBits(Bits.size()), RHSBits(Bits.size()); | |||||
990 | getValueBits(V.getOperand(0), LHSBits); | |||||
991 | getValueBits(V.getOperand(1), RHSBits); | |||||
992 | ||||||
993 | bool AllDisjoint = true; | |||||
994 | for (unsigned i = 0; i < Bits.size(); ++i) | |||||
995 | if (LHSBits[i].isZero()) | |||||
996 | Bits[i] = RHSBits[i]; | |||||
997 | else if (RHSBits[i].isZero()) | |||||
998 | Bits[i] = LHSBits[i]; | |||||
999 | else { | |||||
1000 | AllDisjoint = false; | |||||
1001 | break; | |||||
1002 | } | |||||
1003 | ||||||
1004 | if (!AllDisjoint) | |||||
1005 | break; | |||||
1006 | ||||||
1007 | return true; | |||||
1008 | } | |||||
1009 | } | |||||
1010 | ||||||
1011 | for (unsigned i = 0; i < Bits.size(); ++i) | |||||
1012 | Bits[i] = ValueBit(V, i); | |||||
1013 | ||||||
1014 | return false; | |||||
1015 | } | |||||
1016 | ||||||
1017 | // For each value (except the constant ones), compute the left-rotate amount | |||||
1018 | // to get it from its original to final position. | |||||
1019 | void computeRotationAmounts() { | |||||
1020 | HasZeros = false; | |||||
1021 | RLAmt.resize(Bits.size()); | |||||
1022 | for (unsigned i = 0; i < Bits.size(); ++i) | |||||
1023 | if (Bits[i].hasValue()) { | |||||
1024 | unsigned VBI = Bits[i].getValueBitIndex(); | |||||
1025 | if (i >= VBI) | |||||
1026 | RLAmt[i] = i - VBI; | |||||
1027 | else | |||||
1028 | RLAmt[i] = Bits.size() - (VBI - i); | |||||
1029 | } else if (Bits[i].isZero()) { | |||||
1030 | HasZeros = true; | |||||
1031 | RLAmt[i] = UINT32_MAX(4294967295U); | |||||
1032 | } else { | |||||
1033 | llvm_unreachable("Unknown value bit type")::llvm::llvm_unreachable_internal("Unknown value bit type", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1033); | |||||
1034 | } | |||||
1035 | } | |||||
1036 | ||||||
1037 | // Collect groups of consecutive bits with the same underlying value and | |||||
1038 | // rotation factor. If we're doing late masking, we ignore zeros, otherwise | |||||
1039 | // they break up groups. | |||||
1040 | void collectBitGroups(bool LateMask) { | |||||
1041 | BitGroups.clear(); | |||||
1042 | ||||||
1043 | unsigned LastRLAmt = RLAmt[0]; | |||||
1044 | SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue(); | |||||
1045 | unsigned LastGroupStartIdx = 0; | |||||
1046 | for (unsigned i = 1; i < Bits.size(); ++i) { | |||||
1047 | unsigned ThisRLAmt = RLAmt[i]; | |||||
1048 | SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue(); | |||||
1049 | if (LateMask && !ThisValue) { | |||||
1050 | ThisValue = LastValue; | |||||
1051 | ThisRLAmt = LastRLAmt; | |||||
1052 | // If we're doing late masking, then the first bit group always starts | |||||
1053 | // at zero (even if the first bits were zero). | |||||
1054 | if (BitGroups.empty()) | |||||
1055 | LastGroupStartIdx = 0; | |||||
1056 | } | |||||
1057 | ||||||
1058 | // If this bit has the same underlying value and the same rotate factor as | |||||
1059 | // the last one, then they're part of the same group. | |||||
1060 | if (ThisRLAmt == LastRLAmt && ThisValue == LastValue) | |||||
1061 | continue; | |||||
1062 | ||||||
1063 | if (LastValue.getNode()) | |||||
1064 | BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, | |||||
1065 | i-1)); | |||||
1066 | LastRLAmt = ThisRLAmt; | |||||
1067 | LastValue = ThisValue; | |||||
1068 | LastGroupStartIdx = i; | |||||
1069 | } | |||||
1070 | if (LastValue.getNode()) | |||||
1071 | BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, | |||||
1072 | Bits.size()-1)); | |||||
1073 | ||||||
1074 | if (BitGroups.empty()) | |||||
1075 | return; | |||||
1076 | ||||||
1077 | // We might be able to combine the first and last groups. | |||||
1078 | if (BitGroups.size() > 1) { | |||||
1079 | // If the first and last groups are the same, then remove the first group | |||||
1080 | // in favor of the last group, making the ending index of the last group | |||||
1081 | // equal to the ending index of the to-be-removed first group. | |||||
1082 | if (BitGroups[0].StartIdx == 0 && | |||||
1083 | BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 && | |||||
1084 | BitGroups[0].V == BitGroups[BitGroups.size()-1].V && | |||||
1085 | BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) { | |||||
1086 | DEBUG(dbgs() << "\tcombining final bit group with initial one\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining final bit group with initial one\n" ; } } while (0); | |||||
1087 | BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx; | |||||
1088 | BitGroups.erase(BitGroups.begin()); | |||||
1089 | } | |||||
1090 | } | |||||
1091 | } | |||||
1092 | ||||||
1093 | // Take all (SDValue, RLAmt) pairs and sort them by the number of groups | |||||
1094 | // associated with each. If there is a degeneracy, pick the one that occurs | |||||
1095 | // first (in the final value). | |||||
1096 | void collectValueRotInfo() { | |||||
1097 | ValueRots.clear(); | |||||
1098 | ||||||
1099 | for (auto &BG : BitGroups) { | |||||
1100 | unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0); | |||||
1101 | ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)]; | |||||
1102 | VRI.V = BG.V; | |||||
1103 | VRI.RLAmt = BG.RLAmt; | |||||
1104 | VRI.Repl32 = BG.Repl32; | |||||
1105 | VRI.NumGroups += 1; | |||||
1106 | VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx); | |||||
1107 | } | |||||
1108 | ||||||
1109 | // Now that we've collected the various ValueRotInfo instances, we need to | |||||
1110 | // sort them. | |||||
1111 | ValueRotsVec.clear(); | |||||
1112 | for (auto &I : ValueRots) { | |||||
1113 | ValueRotsVec.push_back(I.second); | |||||
1114 | } | |||||
1115 | std::sort(ValueRotsVec.begin(), ValueRotsVec.end()); | |||||
1116 | } | |||||
1117 | ||||||
1118 | // In 64-bit mode, rlwinm and friends have a rotation operator that | |||||
1119 | // replicates the low-order 32 bits into the high-order 32-bits. The mask | |||||
1120 | // indices of these instructions can only be in the lower 32 bits, so they | |||||
1121 | // can only represent some 64-bit bit groups. However, when they can be used, | |||||
1122 | // the 32-bit replication can be used to represent, as a single bit group, | |||||
1123 | // otherwise separate bit groups. We'll convert to replicated-32-bit bit | |||||
1124 | // groups when possible. Returns true if any of the bit groups were | |||||
1125 | // converted. | |||||
1126 | void assignRepl32BitGroups() { | |||||
1127 | // If we have bits like this: | |||||
1128 | // | |||||
1129 | // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 | |||||
1130 | // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24 | |||||
1131 | // Groups: | RLAmt = 8 | RLAmt = 40 | | |||||
1132 | // | |||||
1133 | // But, making use of a 32-bit operation that replicates the low-order 32 | |||||
1134 | // bits into the high-order 32 bits, this can be one bit group with a RLAmt | |||||
1135 | // of 8. | |||||
1136 | ||||||
1137 | auto IsAllLow32 = [this](BitGroup & BG) { | |||||
1138 | if (BG.StartIdx <= BG.EndIdx) { | |||||
1139 | for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) { | |||||
1140 | if (!Bits[i].hasValue()) | |||||
1141 | continue; | |||||
1142 | if (Bits[i].getValueBitIndex() >= 32) | |||||
1143 | return false; | |||||
1144 | } | |||||
1145 | } else { | |||||
1146 | for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) { | |||||
1147 | if (!Bits[i].hasValue()) | |||||
1148 | continue; | |||||
1149 | if (Bits[i].getValueBitIndex() >= 32) | |||||
1150 | return false; | |||||
1151 | } | |||||
1152 | for (unsigned i = 0; i <= BG.EndIdx; ++i) { | |||||
1153 | if (!Bits[i].hasValue()) | |||||
1154 | continue; | |||||
1155 | if (Bits[i].getValueBitIndex() >= 32) | |||||
1156 | return false; | |||||
1157 | } | |||||
1158 | } | |||||
1159 | ||||||
1160 | return true; | |||||
1161 | }; | |||||
1162 | ||||||
1163 | for (auto &BG : BitGroups) { | |||||
1164 | if (BG.StartIdx < 32 && BG.EndIdx < 32) { | |||||
1165 | if (IsAllLow32(BG)) { | |||||
1166 | if (BG.RLAmt >= 32) { | |||||
1167 | BG.RLAmt -= 32; | |||||
1168 | BG.Repl32CR = true; | |||||
1169 | } | |||||
1170 | ||||||
1171 | BG.Repl32 = true; | |||||
1172 | ||||||
1173 | DEBUG(dbgs() << "\t32-bit replicated bit group for " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t32-bit replicated bit group for " << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" << BG.StartIdx << ", " << BG .EndIdx << "]\n"; } } while (0) | |||||
1174 | BG.V.getNode() << " RLAmt = " << BG.RLAmt <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t32-bit replicated bit group for " << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" << BG.StartIdx << ", " << BG .EndIdx << "]\n"; } } while (0) | |||||
1175 | " [" << BG.StartIdx << ", " << BG.EndIdx << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t32-bit replicated bit group for " << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" << BG.StartIdx << ", " << BG .EndIdx << "]\n"; } } while (0); | |||||
1176 | } | |||||
1177 | } | |||||
1178 | } | |||||
1179 | ||||||
1180 | // Now walk through the bit groups, consolidating where possible. | |||||
1181 | for (auto I = BitGroups.begin(); I != BitGroups.end();) { | |||||
1182 | // We might want to remove this bit group by merging it with the previous | |||||
1183 | // group (which might be the ending group). | |||||
1184 | auto IP = (I == BitGroups.begin()) ? | |||||
1185 | std::prev(BitGroups.end()) : std::prev(I); | |||||
1186 | if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt && | |||||
1187 | I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) { | |||||
1188 | ||||||
1189 | DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with group with range [" << IP ->StartIdx << ", " << IP->EndIdx << "]\n" ; } } while (0) | |||||
1190 | I->V.getNode() << " RLAmt = " << I->RLAmt <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with group with range [" << IP ->StartIdx << ", " << IP->EndIdx << "]\n" ; } } while (0) | |||||
1191 | " [" << I->StartIdx << ", " << I->EndIdx <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with group with range [" << IP ->StartIdx << ", " << IP->EndIdx << "]\n" ; } } while (0) | |||||
1192 | "] with group with range [" <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with group with range [" << IP ->StartIdx << ", " << IP->EndIdx << "]\n" ; } } while (0) | |||||
1193 | IP->StartIdx << ", " << IP->EndIdx << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining 32-bit replicated bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with group with range [" << IP ->StartIdx << ", " << IP->EndIdx << "]\n" ; } } while (0); | |||||
1194 | ||||||
1195 | IP->EndIdx = I->EndIdx; | |||||
1196 | IP->Repl32CR = IP->Repl32CR || I->Repl32CR; | |||||
1197 | IP->Repl32Coalesced = true; | |||||
1198 | I = BitGroups.erase(I); | |||||
1199 | continue; | |||||
1200 | } else { | |||||
1201 | // There is a special case worth handling: If there is a single group | |||||
1202 | // covering the entire upper 32 bits, and it can be merged with both | |||||
1203 | // the next and previous groups (which might be the same group), then | |||||
1204 | // do so. If it is the same group (so there will be only one group in | |||||
1205 | // total), then we need to reverse the order of the range so that it | |||||
1206 | // covers the entire 64 bits. | |||||
1207 | if (I->StartIdx == 32 && I->EndIdx == 63) { | |||||
1208 | assert(std::next(I) == BitGroups.end() &&((std::next(I) == BitGroups.end() && "bit group ends at index 63 but there is another?" ) ? static_cast<void> (0) : __assert_fail ("std::next(I) == BitGroups.end() && \"bit group ends at index 63 but there is another?\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1209, __PRETTY_FUNCTION__)) | |||||
1209 | "bit group ends at index 63 but there is another?")((std::next(I) == BitGroups.end() && "bit group ends at index 63 but there is another?" ) ? static_cast<void> (0) : __assert_fail ("std::next(I) == BitGroups.end() && \"bit group ends at index 63 but there is another?\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1209, __PRETTY_FUNCTION__)); | |||||
1210 | auto IN = BitGroups.begin(); | |||||
1211 | ||||||
1212 | if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V && | |||||
1213 | (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt && | |||||
1214 | IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP && | |||||
1215 | IsAllLow32(*I)) { | |||||
1216 | ||||||
1217 | DEBUG(dbgs() << "\tcombining bit group for " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with 32-bit replicated groups with ranges [" << IP->StartIdx << ", " << IP->EndIdx << "] and [" << IN->StartIdx << ", " << IN->EndIdx << "]\n"; } } while (0) | |||||
1218 | I->V.getNode() << " RLAmt = " << I->RLAmt <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with 32-bit replicated groups with ranges [" << IP->StartIdx << ", " << IP->EndIdx << "] and [" << IN->StartIdx << ", " << IN->EndIdx << "]\n"; } } while (0) | |||||
1219 | " [" << I->StartIdx << ", " << I->EndIdx <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with 32-bit replicated groups with ranges [" << IP->StartIdx << ", " << IP->EndIdx << "] and [" << IN->StartIdx << ", " << IN->EndIdx << "]\n"; } } while (0) | |||||
1220 | "] with 32-bit replicated groups with ranges [" <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with 32-bit replicated groups with ranges [" << IP->StartIdx << ", " << IP->EndIdx << "] and [" << IN->StartIdx << ", " << IN->EndIdx << "]\n"; } } while (0) | |||||
1221 | IP->StartIdx << ", " << IP->EndIdx << "] and [" <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with 32-bit replicated groups with ranges [" << IP->StartIdx << ", " << IP->EndIdx << "] and [" << IN->StartIdx << ", " << IN->EndIdx << "]\n"; } } while (0) | |||||
1222 | IN->StartIdx << ", " << IN->EndIdx << "]\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tcombining bit group for " << I->V.getNode() << " RLAmt = " << I-> RLAmt << " [" << I->StartIdx << ", " << I->EndIdx << "] with 32-bit replicated groups with ranges [" << IP->StartIdx << ", " << IP->EndIdx << "] and [" << IN->StartIdx << ", " << IN->EndIdx << "]\n"; } } while (0); | |||||
1223 | ||||||
1224 | if (IP == IN) { | |||||
1225 | // There is only one other group; change it to cover the whole | |||||
1226 | // range (backward, so that it can still be Repl32 but cover the | |||||
1227 | // whole 64-bit range). | |||||
1228 | IP->StartIdx = 31; | |||||
1229 | IP->EndIdx = 30; | |||||
1230 | IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32; | |||||
1231 | IP->Repl32Coalesced = true; | |||||
1232 | I = BitGroups.erase(I); | |||||
1233 | } else { | |||||
1234 | // There are two separate groups, one before this group and one | |||||
1235 | // after us (at the beginning). We're going to remove this group, | |||||
1236 | // but also the group at the very beginning. | |||||
1237 | IP->EndIdx = IN->EndIdx; | |||||
1238 | IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32; | |||||
1239 | IP->Repl32Coalesced = true; | |||||
1240 | I = BitGroups.erase(I); | |||||
1241 | BitGroups.erase(BitGroups.begin()); | |||||
1242 | } | |||||
1243 | ||||||
1244 | // This must be the last group in the vector (and we might have | |||||
1245 | // just invalidated the iterator above), so break here. | |||||
1246 | break; | |||||
1247 | } | |||||
1248 | } | |||||
1249 | } | |||||
1250 | ||||||
1251 | ++I; | |||||
1252 | } | |||||
1253 | } | |||||
1254 | ||||||
1255 | SDValue getI32Imm(unsigned Imm, SDLoc dl) { | |||||
1256 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); | |||||
1257 | } | |||||
1258 | ||||||
1259 | uint64_t getZerosMask() { | |||||
1260 | uint64_t Mask = 0; | |||||
1261 | for (unsigned i = 0; i < Bits.size(); ++i) { | |||||
1262 | if (Bits[i].hasValue()) | |||||
1263 | continue; | |||||
1264 | Mask |= (UINT64_C(1)1UL << i); | |||||
1265 | } | |||||
1266 | ||||||
1267 | return ~Mask; | |||||
1268 | } | |||||
1269 | ||||||
1270 | // Depending on the number of groups for a particular value, it might be | |||||
1271 | // better to rotate, mask explicitly (using andi/andis), and then or the | |||||
1272 | // result. Select this part of the result first. | |||||
1273 | void SelectAndParts32(SDLoc dl, SDValue &Res, unsigned *InstCnt) { | |||||
1274 | if (BPermRewriterNoMasking) | |||||
1275 | return; | |||||
1276 | ||||||
1277 | for (ValueRotInfo &VRI : ValueRotsVec) { | |||||
1278 | unsigned Mask = 0; | |||||
1279 | for (unsigned i = 0; i < Bits.size(); ++i) { | |||||
1280 | if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V) | |||||
1281 | continue; | |||||
1282 | if (RLAmt[i] != VRI.RLAmt) | |||||
1283 | continue; | |||||
1284 | Mask |= (1u << i); | |||||
1285 | } | |||||
1286 | ||||||
1287 | // Compute the masks for andi/andis that would be necessary. | |||||
1288 | unsigned ANDIMask = (Mask & UINT16_MAX(65535)), ANDISMask = Mask >> 16; | |||||
1289 | assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask for value bit groups" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask for value bit groups\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1290, __PRETTY_FUNCTION__)) | |||||
1290 | "No set bits in mask for value bit groups")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask for value bit groups" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask for value bit groups\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1290, __PRETTY_FUNCTION__)); | |||||
1291 | bool NeedsRotate = VRI.RLAmt != 0; | |||||
1292 | ||||||
1293 | // We're trying to minimize the number of instructions. If we have one | |||||
1294 | // group, using one of andi/andis can break even. If we have three | |||||
1295 | // groups, we can use both andi and andis and break even (to use both | |||||
1296 | // andi and andis we also need to or the results together). We need four | |||||
1297 | // groups if we also need to rotate. To use andi/andis we need to do more | |||||
1298 | // than break even because rotate-and-mask instructions tend to be easier | |||||
1299 | // to schedule. | |||||
1300 | ||||||
1301 | // FIXME: We've biased here against using andi/andis, which is right for | |||||
1302 | // POWER cores, but not optimal everywhere. For example, on the A2, | |||||
1303 | // andi/andis have single-cycle latency whereas the rotate-and-mask | |||||
1304 | // instructions take two cycles, and it would be better to bias toward | |||||
1305 | // andi/andis in break-even cases. | |||||
1306 | ||||||
1307 | unsigned NumAndInsts = (unsigned) NeedsRotate + | |||||
1308 | (unsigned) (ANDIMask != 0) + | |||||
1309 | (unsigned) (ANDISMask != 0) + | |||||
1310 | (unsigned) (ANDIMask != 0 && ANDISMask != 0) + | |||||
1311 | (unsigned) (bool) Res; | |||||
1312 | ||||||
1313 | DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << ":" << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << VRI.NumGroups << "\n"; } } while (0) | |||||
1314 | " RL: " << VRI.RLAmt << ":" <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << ":" << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << VRI.NumGroups << "\n"; } } while (0) | |||||
1315 | "\n\t\t\tisel using masking: " << NumAndInsts <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << ":" << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << VRI.NumGroups << "\n"; } } while (0) | |||||
1316 | " using rotates: " << VRI.NumGroups << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << ":" << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << VRI.NumGroups << "\n"; } } while (0); | |||||
1317 | ||||||
1318 | if (NumAndInsts >= VRI.NumGroups) | |||||
1319 | continue; | |||||
1320 | ||||||
1321 | DEBUG(dbgs() << "\t\t\t\tusing masking\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\t\t\tusing masking\n"; } } while (0); | |||||
1322 | ||||||
1323 | if (InstCnt) *InstCnt += NumAndInsts; | |||||
1324 | ||||||
1325 | SDValue VRot; | |||||
1326 | if (VRI.RLAmt) { | |||||
1327 | SDValue Ops[] = | |||||
1328 | { VRI.V, getI32Imm(VRI.RLAmt, dl), getI32Imm(0, dl), | |||||
1329 | getI32Imm(31, dl) }; | |||||
1330 | VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, | |||||
1331 | Ops), 0); | |||||
1332 | } else { | |||||
1333 | VRot = VRI.V; | |||||
1334 | } | |||||
1335 | ||||||
1336 | SDValue ANDIVal, ANDISVal; | |||||
1337 | if (ANDIMask != 0) | |||||
1338 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo, dl, MVT::i32, | |||||
1339 | VRot, getI32Imm(ANDIMask, dl)), 0); | |||||
1340 | if (ANDISMask != 0) | |||||
1341 | ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo, dl, MVT::i32, | |||||
1342 | VRot, getI32Imm(ANDISMask, dl)), 0); | |||||
1343 | ||||||
1344 | SDValue TotalVal; | |||||
1345 | if (!ANDIVal) | |||||
1346 | TotalVal = ANDISVal; | |||||
1347 | else if (!ANDISVal) | |||||
1348 | TotalVal = ANDIVal; | |||||
1349 | else | |||||
1350 | TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, | |||||
1351 | ANDIVal, ANDISVal), 0); | |||||
1352 | ||||||
1353 | if (!Res) | |||||
1354 | Res = TotalVal; | |||||
1355 | else | |||||
1356 | Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, | |||||
1357 | Res, TotalVal), 0); | |||||
1358 | ||||||
1359 | // Now, remove all groups with this underlying value and rotation | |||||
1360 | // factor. | |||||
1361 | eraseMatchingBitGroups([VRI](const BitGroup &BG) { | |||||
1362 | return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; | |||||
1363 | }); | |||||
1364 | } | |||||
1365 | } | |||||
1366 | ||||||
1367 | // Instruction selection for the 32-bit case. | |||||
1368 | SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) { | |||||
1369 | SDLoc dl(N); | |||||
1370 | SDValue Res; | |||||
1371 | ||||||
1372 | if (InstCnt) *InstCnt = 0; | |||||
1373 | ||||||
1374 | // Take care of cases that should use andi/andis first. | |||||
1375 | SelectAndParts32(dl, Res, InstCnt); | |||||
1376 | ||||||
1377 | // If we've not yet selected a 'starting' instruction, and we have no zeros | |||||
1378 | // to fill in, select the (Value, RLAmt) with the highest priority (largest | |||||
1379 | // number of groups), and start with this rotated value. | |||||
1380 | if ((!HasZeros || LateMask) && !Res) { | |||||
1381 | ValueRotInfo &VRI = ValueRotsVec[0]; | |||||
1382 | if (VRI.RLAmt) { | |||||
1383 | if (InstCnt) *InstCnt += 1; | |||||
1384 | SDValue Ops[] = | |||||
1385 | { VRI.V, getI32Imm(VRI.RLAmt, dl), getI32Imm(0, dl), | |||||
1386 | getI32Imm(31, dl) }; | |||||
1387 | Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), | |||||
1388 | 0); | |||||
1389 | } else { | |||||
1390 | Res = VRI.V; | |||||
1391 | } | |||||
1392 | ||||||
1393 | // Now, remove all groups with this underlying value and rotation factor. | |||||
1394 | eraseMatchingBitGroups([VRI](const BitGroup &BG) { | |||||
1395 | return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; | |||||
1396 | }); | |||||
1397 | } | |||||
1398 | ||||||
1399 | if (InstCnt) *InstCnt += BitGroups.size(); | |||||
1400 | ||||||
1401 | // Insert the other groups (one at a time). | |||||
1402 | for (auto &BG : BitGroups) { | |||||
1403 | if (!Res) { | |||||
1404 | SDValue Ops[] = | |||||
1405 | { BG.V, getI32Imm(BG.RLAmt, dl), | |||||
1406 | getI32Imm(Bits.size() - BG.EndIdx - 1, dl), | |||||
1407 | getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; | |||||
1408 | Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); | |||||
1409 | } else { | |||||
1410 | SDValue Ops[] = | |||||
1411 | { Res, BG.V, getI32Imm(BG.RLAmt, dl), | |||||
1412 | getI32Imm(Bits.size() - BG.EndIdx - 1, dl), | |||||
1413 | getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; | |||||
1414 | Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0); | |||||
1415 | } | |||||
1416 | } | |||||
1417 | ||||||
1418 | if (LateMask) { | |||||
1419 | unsigned Mask = (unsigned) getZerosMask(); | |||||
1420 | ||||||
1421 | unsigned ANDIMask = (Mask & UINT16_MAX(65535)), ANDISMask = Mask >> 16; | |||||
1422 | assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in zeros mask?" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in zeros mask?\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1423, __PRETTY_FUNCTION__)) | |||||
1423 | "No set bits in zeros mask?")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in zeros mask?" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in zeros mask?\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1423, __PRETTY_FUNCTION__)); | |||||
1424 | ||||||
1425 | if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + | |||||
1426 | (unsigned) (ANDISMask != 0) + | |||||
1427 | (unsigned) (ANDIMask != 0 && ANDISMask != 0); | |||||
1428 | ||||||
1429 | SDValue ANDIVal, ANDISVal; | |||||
1430 | if (ANDIMask != 0) | |||||
1431 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo, dl, MVT::i32, | |||||
1432 | Res, getI32Imm(ANDIMask, dl)), 0); | |||||
1433 | if (ANDISMask != 0) | |||||
1434 | ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo, dl, MVT::i32, | |||||
1435 | Res, getI32Imm(ANDISMask, dl)), 0); | |||||
1436 | ||||||
1437 | if (!ANDIVal) | |||||
1438 | Res = ANDISVal; | |||||
1439 | else if (!ANDISVal) | |||||
1440 | Res = ANDIVal; | |||||
1441 | else | |||||
1442 | Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, | |||||
1443 | ANDIVal, ANDISVal), 0); | |||||
1444 | } | |||||
1445 | ||||||
1446 | return Res.getNode(); | |||||
1447 | } | |||||
1448 | ||||||
1449 | unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32, | |||||
1450 | unsigned MaskStart, unsigned MaskEnd, | |||||
1451 | bool IsIns) { | |||||
1452 | // In the notation used by the instructions, 'start' and 'end' are reversed | |||||
1453 | // because bits are counted from high to low order. | |||||
1454 | unsigned InstMaskStart = 64 - MaskEnd - 1, | |||||
1455 | InstMaskEnd = 64 - MaskStart - 1; | |||||
1456 | ||||||
1457 | if (Repl32) | |||||
1458 | return 1; | |||||
1459 | ||||||
1460 | if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) || | |||||
1461 | InstMaskEnd == 63 - RLAmt) | |||||
1462 | return 1; | |||||
1463 | ||||||
1464 | return 2; | |||||
1465 | } | |||||
1466 | ||||||
1467 | // For 64-bit values, not all combinations of rotates and masks are | |||||
1468 | // available. Produce one if it is available. | |||||
1469 | SDValue SelectRotMask64(SDValue V, SDLoc dl, unsigned RLAmt, bool Repl32, | |||||
1470 | unsigned MaskStart, unsigned MaskEnd, | |||||
1471 | unsigned *InstCnt = nullptr) { | |||||
1472 | // In the notation used by the instructions, 'start' and 'end' are reversed | |||||
1473 | // because bits are counted from high to low order. | |||||
1474 | unsigned InstMaskStart = 64 - MaskEnd - 1, | |||||
1475 | InstMaskEnd = 64 - MaskStart - 1; | |||||
1476 | ||||||
1477 | if (InstCnt) *InstCnt += 1; | |||||
1478 | ||||||
1479 | if (Repl32) { | |||||
1480 | // This rotation amount assumes that the lower 32 bits of the quantity | |||||
1481 | // are replicated in the high 32 bits by the rotation operator (which is | |||||
1482 | // done by rlwinm and friends). | |||||
1483 | assert(InstMaskStart >= 32 && "Mask cannot start out of range")((InstMaskStart >= 32 && "Mask cannot start out of range" ) ? static_cast<void> (0) : __assert_fail ("InstMaskStart >= 32 && \"Mask cannot start out of range\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1483, __PRETTY_FUNCTION__)); | |||||
1484 | assert(InstMaskEnd >= 32 && "Mask cannot end out of range")((InstMaskEnd >= 32 && "Mask cannot end out of range" ) ? static_cast<void> (0) : __assert_fail ("InstMaskEnd >= 32 && \"Mask cannot end out of range\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1484, __PRETTY_FUNCTION__)); | |||||
1485 | SDValue Ops[] = | |||||
1486 | { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart - 32, dl), | |||||
1487 | getI32Imm(InstMaskEnd - 32, dl) }; | |||||
1488 | return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64, | |||||
1489 | Ops), 0); | |||||
1490 | } | |||||
1491 | ||||||
1492 | if (InstMaskEnd == 63) { | |||||
1493 | SDValue Ops[] = | |||||
1494 | { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart, dl) }; | |||||
1495 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0); | |||||
1496 | } | |||||
1497 | ||||||
1498 | if (InstMaskStart == 0) { | |||||
1499 | SDValue Ops[] = | |||||
1500 | { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskEnd, dl) }; | |||||
1501 | return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0); | |||||
1502 | } | |||||
1503 | ||||||
1504 | if (InstMaskEnd == 63 - RLAmt) { | |||||
1505 | SDValue Ops[] = | |||||
1506 | { V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart, dl) }; | |||||
1507 | return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0); | |||||
1508 | } | |||||
1509 | ||||||
1510 | // We cannot do this with a single instruction, so we'll use two. The | |||||
1511 | // problem is that we're not free to choose both a rotation amount and mask | |||||
1512 | // start and end independently. We can choose an arbitrary mask start and | |||||
1513 | // end, but then the rotation amount is fixed. Rotation, however, can be | |||||
1514 | // inverted, and so by applying an "inverse" rotation first, we can get the | |||||
1515 | // desired result. | |||||
1516 | if (InstCnt) *InstCnt += 1; | |||||
1517 | ||||||
1518 | // The rotation mask for the second instruction must be MaskStart. | |||||
1519 | unsigned RLAmt2 = MaskStart; | |||||
1520 | // The first instruction must rotate V so that the overall rotation amount | |||||
1521 | // is RLAmt. | |||||
1522 | unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; | |||||
1523 | if (RLAmt1) | |||||
1524 | V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); | |||||
1525 | return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd); | |||||
1526 | } | |||||
1527 | ||||||
1528 | // For 64-bit values, not all combinations of rotates and masks are | |||||
1529 | // available. Produce a rotate-mask-and-insert if one is available. | |||||
1530 | SDValue SelectRotMaskIns64(SDValue Base, SDValue V, SDLoc dl, unsigned RLAmt, | |||||
1531 | bool Repl32, unsigned MaskStart, | |||||
1532 | unsigned MaskEnd, unsigned *InstCnt = nullptr) { | |||||
1533 | // In the notation used by the instructions, 'start' and 'end' are reversed | |||||
1534 | // because bits are counted from high to low order. | |||||
1535 | unsigned InstMaskStart = 64 - MaskEnd - 1, | |||||
1536 | InstMaskEnd = 64 - MaskStart - 1; | |||||
1537 | ||||||
1538 | if (InstCnt) *InstCnt += 1; | |||||
1539 | ||||||
1540 | if (Repl32) { | |||||
1541 | // This rotation amount assumes that the lower 32 bits of the quantity | |||||
1542 | // are replicated in the high 32 bits by the rotation operator (which is | |||||
1543 | // done by rlwinm and friends). | |||||
1544 | assert(InstMaskStart >= 32 && "Mask cannot start out of range")((InstMaskStart >= 32 && "Mask cannot start out of range" ) ? static_cast<void> (0) : __assert_fail ("InstMaskStart >= 32 && \"Mask cannot start out of range\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1544, __PRETTY_FUNCTION__)); | |||||
1545 | assert(InstMaskEnd >= 32 && "Mask cannot end out of range")((InstMaskEnd >= 32 && "Mask cannot end out of range" ) ? static_cast<void> (0) : __assert_fail ("InstMaskEnd >= 32 && \"Mask cannot end out of range\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1545, __PRETTY_FUNCTION__)); | |||||
1546 | SDValue Ops[] = | |||||
1547 | { Base, V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart - 32, dl), | |||||
1548 | getI32Imm(InstMaskEnd - 32, dl) }; | |||||
1549 | return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, | |||||
1550 | Ops), 0); | |||||
1551 | } | |||||
1552 | ||||||
1553 | if (InstMaskEnd == 63 - RLAmt) { | |||||
1554 | SDValue Ops[] = | |||||
1555 | { Base, V, getI32Imm(RLAmt, dl), getI32Imm(InstMaskStart, dl) }; | |||||
1556 | return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0); | |||||
1557 | } | |||||
1558 | ||||||
1559 | // We cannot do this with a single instruction, so we'll use two. The | |||||
1560 | // problem is that we're not free to choose both a rotation amount and mask | |||||
1561 | // start and end independently. We can choose an arbitrary mask start and | |||||
1562 | // end, but then the rotation amount is fixed. Rotation, however, can be | |||||
1563 | // inverted, and so by applying an "inverse" rotation first, we can get the | |||||
1564 | // desired result. | |||||
1565 | if (InstCnt) *InstCnt += 1; | |||||
1566 | ||||||
1567 | // The rotation mask for the second instruction must be MaskStart. | |||||
1568 | unsigned RLAmt2 = MaskStart; | |||||
1569 | // The first instruction must rotate V so that the overall rotation amount | |||||
1570 | // is RLAmt. | |||||
1571 | unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; | |||||
1572 | if (RLAmt1) | |||||
1573 | V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); | |||||
1574 | return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd); | |||||
1575 | } | |||||
1576 | ||||||
1577 | void SelectAndParts64(SDLoc dl, SDValue &Res, unsigned *InstCnt) { | |||||
1578 | if (BPermRewriterNoMasking) | |||||
1579 | return; | |||||
1580 | ||||||
1581 | // The idea here is the same as in the 32-bit version, but with additional | |||||
1582 | // complications from the fact that Repl32 might be true. Because we | |||||
1583 | // aggressively convert bit groups to Repl32 form (which, for small | |||||
1584 | // rotation factors, involves no other change), and then coalesce, it might | |||||
1585 | // be the case that a single 64-bit masking operation could handle both | |||||
1586 | // some Repl32 groups and some non-Repl32 groups. If converting to Repl32 | |||||
1587 | // form allowed coalescing, then we must use a 32-bit rotaton in order to | |||||
1588 | // completely capture the new combined bit group. | |||||
1589 | ||||||
1590 | for (ValueRotInfo &VRI : ValueRotsVec) { | |||||
1591 | uint64_t Mask = 0; | |||||
1592 | ||||||
1593 | // We need to add to the mask all bits from the associated bit groups. | |||||
1594 | // If Repl32 is false, we need to add bits from bit groups that have | |||||
1595 | // Repl32 true, but are trivially convertable to Repl32 false. Such a | |||||
1596 | // group is trivially convertable if it overlaps only with the lower 32 | |||||
1597 | // bits, and the group has not been coalesced. | |||||
1598 | auto MatchingBG = [VRI](const BitGroup &BG) { | |||||
1599 | if (VRI.V != BG.V) | |||||
1600 | return false; | |||||
1601 | ||||||
1602 | unsigned EffRLAmt = BG.RLAmt; | |||||
1603 | if (!VRI.Repl32 && BG.Repl32) { | |||||
1604 | if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx && | |||||
1605 | !BG.Repl32Coalesced) { | |||||
1606 | if (BG.Repl32CR) | |||||
1607 | EffRLAmt += 32; | |||||
1608 | } else { | |||||
1609 | return false; | |||||
1610 | } | |||||
1611 | } else if (VRI.Repl32 != BG.Repl32) { | |||||
1612 | return false; | |||||
1613 | } | |||||
1614 | ||||||
1615 | return VRI.RLAmt == EffRLAmt; | |||||
1616 | }; | |||||
1617 | ||||||
1618 | for (auto &BG : BitGroups) { | |||||
1619 | if (!MatchingBG(BG)) | |||||
1620 | continue; | |||||
1621 | ||||||
1622 | if (BG.StartIdx <= BG.EndIdx) { | |||||
1623 | for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) | |||||
1624 | Mask |= (UINT64_C(1)1UL << i); | |||||
1625 | } else { | |||||
1626 | for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) | |||||
1627 | Mask |= (UINT64_C(1)1UL << i); | |||||
1628 | for (unsigned i = 0; i <= BG.EndIdx; ++i) | |||||
1629 | Mask |= (UINT64_C(1)1UL << i); | |||||
1630 | } | |||||
1631 | } | |||||
1632 | ||||||
1633 | // We can use the 32-bit andi/andis technique if the mask does not | |||||
1634 | // require any higher-order bits. This can save an instruction compared | |||||
1635 | // to always using the general 64-bit technique. | |||||
1636 | bool Use32BitInsts = isUInt<32>(Mask); | |||||
1637 | // Compute the masks for andi/andis that would be necessary. | |||||
1638 | unsigned ANDIMask = (Mask & UINT16_MAX(65535)), | |||||
1639 | ANDISMask = (Mask >> 16) & UINT16_MAX(65535); | |||||
1640 | ||||||
1641 | bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)); | |||||
1642 | ||||||
1643 | unsigned NumAndInsts = (unsigned) NeedsRotate + | |||||
1644 | (unsigned) (bool) Res; | |||||
1645 | if (Use32BitInsts) | |||||
1646 | NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) + | |||||
1647 | (unsigned) (ANDIMask != 0 && ANDISMask != 0); | |||||
1648 | else | |||||
1649 | NumAndInsts += SelectInt64Count(Mask) + /* and */ 1; | |||||
1650 | ||||||
1651 | unsigned NumRLInsts = 0; | |||||
1652 | bool FirstBG = true; | |||||
1653 | for (auto &BG : BitGroups) { | |||||
1654 | if (!MatchingBG(BG)) | |||||
1655 | continue; | |||||
1656 | NumRLInsts += | |||||
1657 | SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx, | |||||
1658 | !FirstBG); | |||||
1659 | FirstBG = false; | |||||
1660 | } | |||||
1661 | ||||||
1662 | DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << NumRLInsts << "\n"; } } while (0) | |||||
1663 | " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << NumRLInsts << "\n"; } } while (0) | |||||
1664 | "\n\t\t\tisel using masking: " << NumAndInsts <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << NumRLInsts << "\n"; } } while (0) | |||||
1665 | " using rotates: " << NumRLInsts << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\trotation groups for " << VRI.V.getNode() << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") << "\n\t\t\tisel using masking: " << NumAndInsts << " using rotates: " << NumRLInsts << "\n"; } } while (0); | |||||
1666 | ||||||
1667 | // When we'd use andi/andis, we bias toward using the rotates (andi only | |||||
1668 | // has a record form, and is cracked on POWER cores). However, when using | |||||
1669 | // general 64-bit constant formation, bias toward the constant form, | |||||
1670 | // because that exposes more opportunities for CSE. | |||||
1671 | if (NumAndInsts > NumRLInsts) | |||||
1672 | continue; | |||||
1673 | if (Use32BitInsts && NumAndInsts == NumRLInsts) | |||||
1674 | continue; | |||||
1675 | ||||||
1676 | DEBUG(dbgs() << "\t\t\t\tusing masking\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\t\t\tusing masking\n"; } } while (0); | |||||
1677 | ||||||
1678 | if (InstCnt) *InstCnt += NumAndInsts; | |||||
1679 | ||||||
1680 | SDValue VRot; | |||||
1681 | // We actually need to generate a rotation if we have a non-zero rotation | |||||
1682 | // factor or, in the Repl32 case, if we care about any of the | |||||
1683 | // higher-order replicated bits. In the latter case, we generate a mask | |||||
1684 | // backward so that it actually includes the entire 64 bits. | |||||
1685 | if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask))) | |||||
1686 | VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, | |||||
1687 | VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63); | |||||
1688 | else | |||||
1689 | VRot = VRI.V; | |||||
1690 | ||||||
1691 | SDValue TotalVal; | |||||
1692 | if (Use32BitInsts) { | |||||
1693 | assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1694, __PRETTY_FUNCTION__)) | |||||
1694 | "No set bits in mask when using 32-bit ands for 64-bit value")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1694, __PRETTY_FUNCTION__)); | |||||
1695 | ||||||
1696 | SDValue ANDIVal, ANDISVal; | |||||
1697 | if (ANDIMask != 0) | |||||
1698 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo8, dl, MVT::i64, | |||||
1699 | VRot, getI32Imm(ANDIMask, dl)), 0); | |||||
1700 | if (ANDISMask != 0) | |||||
1701 | ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo8, dl, MVT::i64, | |||||
1702 | VRot, getI32Imm(ANDISMask, dl)), 0); | |||||
1703 | ||||||
1704 | if (!ANDIVal) | |||||
1705 | TotalVal = ANDISVal; | |||||
1706 | else if (!ANDISVal) | |||||
1707 | TotalVal = ANDIVal; | |||||
1708 | else | |||||
1709 | TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, | |||||
1710 | ANDIVal, ANDISVal), 0); | |||||
1711 | } else { | |||||
1712 | TotalVal = SDValue(SelectInt64(CurDAG, dl, Mask), 0); | |||||
1713 | TotalVal = | |||||
1714 | SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, | |||||
1715 | VRot, TotalVal), 0); | |||||
1716 | } | |||||
1717 | ||||||
1718 | if (!Res) | |||||
1719 | Res = TotalVal; | |||||
1720 | else | |||||
1721 | Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, | |||||
1722 | Res, TotalVal), 0); | |||||
1723 | ||||||
1724 | // Now, remove all groups with this underlying value and rotation | |||||
1725 | // factor. | |||||
1726 | eraseMatchingBitGroups(MatchingBG); | |||||
1727 | } | |||||
1728 | } | |||||
1729 | ||||||
1730 | // Instruction selection for the 64-bit case. | |||||
1731 | SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) { | |||||
1732 | SDLoc dl(N); | |||||
1733 | SDValue Res; | |||||
1734 | ||||||
1735 | if (InstCnt) *InstCnt = 0; | |||||
1736 | ||||||
1737 | // Take care of cases that should use andi/andis first. | |||||
1738 | SelectAndParts64(dl, Res, InstCnt); | |||||
1739 | ||||||
1740 | // If we've not yet selected a 'starting' instruction, and we have no zeros | |||||
1741 | // to fill in, select the (Value, RLAmt) with the highest priority (largest | |||||
1742 | // number of groups), and start with this rotated value. | |||||
1743 | if ((!HasZeros || LateMask) && !Res) { | |||||
1744 | // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32 | |||||
1745 | // groups will come first, and so the VRI representing the largest number | |||||
1746 | // of groups might not be first (it might be the first Repl32 groups). | |||||
1747 | unsigned MaxGroupsIdx = 0; | |||||
1748 | if (!ValueRotsVec[0].Repl32) { | |||||
1749 | for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i) | |||||
1750 | if (ValueRotsVec[i].Repl32) { | |||||
1751 | if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups) | |||||
1752 | MaxGroupsIdx = i; | |||||
1753 | break; | |||||
1754 | } | |||||
1755 | } | |||||
1756 | ||||||
1757 | ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx]; | |||||
1758 | bool NeedsRotate = false; | |||||
1759 | if (VRI.RLAmt) { | |||||
1760 | NeedsRotate = true; | |||||
1761 | } else if (VRI.Repl32) { | |||||
1762 | for (auto &BG : BitGroups) { | |||||
1763 | if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt || | |||||
1764 | BG.Repl32 != VRI.Repl32) | |||||
1765 | continue; | |||||
1766 | ||||||
1767 | // We don't need a rotate if the bit group is confined to the lower | |||||
1768 | // 32 bits. | |||||
1769 | if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx) | |||||
1770 | continue; | |||||
1771 | ||||||
1772 | NeedsRotate = true; | |||||
1773 | break; | |||||
1774 | } | |||||
1775 | } | |||||
1776 | ||||||
1777 | if (NeedsRotate) | |||||
1778 | Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, | |||||
1779 | VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63, | |||||
1780 | InstCnt); | |||||
1781 | else | |||||
1782 | Res = VRI.V; | |||||
1783 | ||||||
1784 | // Now, remove all groups with this underlying value and rotation factor. | |||||
1785 | if (Res) | |||||
1786 | eraseMatchingBitGroups([VRI](const BitGroup &BG) { | |||||
1787 | return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt && | |||||
1788 | BG.Repl32 == VRI.Repl32; | |||||
1789 | }); | |||||
1790 | } | |||||
1791 | ||||||
1792 | // Because 64-bit rotates are more flexible than inserts, we might have a | |||||
1793 | // preference regarding which one we do first (to save one instruction). | |||||
1794 | if (!Res) | |||||
1795 | for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) { | |||||
1796 | if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, | |||||
1797 | false) < | |||||
1798 | SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, | |||||
1799 | true)) { | |||||
1800 | if (I != BitGroups.begin()) { | |||||
1801 | BitGroup BG = *I; | |||||
1802 | BitGroups.erase(I); | |||||
1803 | BitGroups.insert(BitGroups.begin(), BG); | |||||
1804 | } | |||||
1805 | ||||||
1806 | break; | |||||
1807 | } | |||||
1808 | } | |||||
1809 | ||||||
1810 | // Insert the other groups (one at a time). | |||||
1811 | for (auto &BG : BitGroups) { | |||||
1812 | if (!Res) | |||||
1813 | Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx, | |||||
1814 | BG.EndIdx, InstCnt); | |||||
1815 | else | |||||
1816 | Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32, | |||||
1817 | BG.StartIdx, BG.EndIdx, InstCnt); | |||||
1818 | } | |||||
1819 | ||||||
1820 | if (LateMask) { | |||||
1821 | uint64_t Mask = getZerosMask(); | |||||
1822 | ||||||
1823 | // We can use the 32-bit andi/andis technique if the mask does not | |||||
1824 | // require any higher-order bits. This can save an instruction compared | |||||
1825 | // to always using the general 64-bit technique. | |||||
1826 | bool Use32BitInsts = isUInt<32>(Mask); | |||||
1827 | // Compute the masks for andi/andis that would be necessary. | |||||
1828 | unsigned ANDIMask = (Mask & UINT16_MAX(65535)), | |||||
1829 | ANDISMask = (Mask >> 16) & UINT16_MAX(65535); | |||||
1830 | ||||||
1831 | if (Use32BitInsts) { | |||||
1832 | assert((ANDIMask != 0 || ANDISMask != 0) &&(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1833, __PRETTY_FUNCTION__)) | |||||
1833 | "No set bits in mask when using 32-bit ands for 64-bit value")(((ANDIMask != 0 || ANDISMask != 0) && "No set bits in mask when using 32-bit ands for 64-bit value" ) ? static_cast<void> (0) : __assert_fail ("(ANDIMask != 0 || ANDISMask != 0) && \"No set bits in mask when using 32-bit ands for 64-bit value\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1833, __PRETTY_FUNCTION__)); | |||||
1834 | ||||||
1835 | if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + | |||||
1836 | (unsigned) (ANDISMask != 0) + | |||||
1837 | (unsigned) (ANDIMask != 0 && ANDISMask != 0); | |||||
1838 | ||||||
1839 | SDValue ANDIVal, ANDISVal; | |||||
1840 | if (ANDIMask != 0) | |||||
1841 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDIo8, dl, MVT::i64, | |||||
1842 | Res, getI32Imm(ANDIMask, dl)), 0); | |||||
1843 | if (ANDISMask != 0) | |||||
1844 | ANDISVal = SDValue(CurDAG->getMachineNode(PPC::ANDISo8, dl, MVT::i64, | |||||
1845 | Res, getI32Imm(ANDISMask, dl)), 0); | |||||
1846 | ||||||
1847 | if (!ANDIVal) | |||||
1848 | Res = ANDISVal; | |||||
1849 | else if (!ANDISVal) | |||||
1850 | Res = ANDIVal; | |||||
1851 | else | |||||
1852 | Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, | |||||
1853 | ANDIVal, ANDISVal), 0); | |||||
1854 | } else { | |||||
1855 | if (InstCnt) *InstCnt += SelectInt64Count(Mask) + /* and */ 1; | |||||
1856 | ||||||
1857 | SDValue MaskVal = SDValue(SelectInt64(CurDAG, dl, Mask), 0); | |||||
1858 | Res = | |||||
1859 | SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, | |||||
1860 | Res, MaskVal), 0); | |||||
1861 | } | |||||
1862 | } | |||||
1863 | ||||||
1864 | return Res.getNode(); | |||||
1865 | } | |||||
1866 | ||||||
1867 | SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) { | |||||
1868 | // Fill in BitGroups. | |||||
1869 | collectBitGroups(LateMask); | |||||
1870 | if (BitGroups.empty()) | |||||
1871 | return nullptr; | |||||
1872 | ||||||
1873 | // For 64-bit values, figure out when we can use 32-bit instructions. | |||||
1874 | if (Bits.size() == 64) | |||||
1875 | assignRepl32BitGroups(); | |||||
1876 | ||||||
1877 | // Fill in ValueRotsVec. | |||||
1878 | collectValueRotInfo(); | |||||
1879 | ||||||
1880 | if (Bits.size() == 32) { | |||||
1881 | return Select32(N, LateMask, InstCnt); | |||||
1882 | } else { | |||||
1883 | assert(Bits.size() == 64 && "Not 64 bits here?")((Bits.size() == 64 && "Not 64 bits here?") ? static_cast <void> (0) : __assert_fail ("Bits.size() == 64 && \"Not 64 bits here?\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 1883, __PRETTY_FUNCTION__)); | |||||
1884 | return Select64(N, LateMask, InstCnt); | |||||
1885 | } | |||||
1886 | ||||||
1887 | return nullptr; | |||||
1888 | } | |||||
1889 | ||||||
1890 | void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) { | |||||
1891 | BitGroups.erase(std::remove_if(BitGroups.begin(), BitGroups.end(), F), | |||||
1892 | BitGroups.end()); | |||||
1893 | } | |||||
1894 | ||||||
1895 | SmallVector<ValueBit, 64> Bits; | |||||
1896 | ||||||
1897 | bool HasZeros; | |||||
1898 | SmallVector<unsigned, 64> RLAmt; | |||||
1899 | ||||||
1900 | SmallVector<BitGroup, 16> BitGroups; | |||||
1901 | ||||||
1902 | DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots; | |||||
1903 | SmallVector<ValueRotInfo, 16> ValueRotsVec; | |||||
1904 | ||||||
1905 | SelectionDAG *CurDAG; | |||||
1906 | ||||||
1907 | public: | |||||
1908 | BitPermutationSelector(SelectionDAG *DAG) | |||||
1909 | : CurDAG(DAG) {} | |||||
1910 | ||||||
1911 | // Here we try to match complex bit permutations into a set of | |||||
1912 | // rotate-and-shift/shift/and/or instructions, using a set of heuristics | |||||
1913 | // known to produce optimial code for common cases (like i32 byte swapping). | |||||
1914 | SDNode *Select(SDNode *N) { | |||||
1915 | Bits.resize(N->getValueType(0).getSizeInBits()); | |||||
1916 | if (!getValueBits(SDValue(N, 0), Bits)) | |||||
1917 | return nullptr; | |||||
1918 | ||||||
1919 | DEBUG(dbgs() << "Considering bit-permutation-based instruction"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Considering bit-permutation-based instruction" " selection for: "; } } while (0) | |||||
1920 | " selection for: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Considering bit-permutation-based instruction" " selection for: "; } } while (0); | |||||
1921 | DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { N->dump(CurDAG); } } while (0); | |||||
1922 | ||||||
1923 | // Fill it RLAmt and set HasZeros. | |||||
1924 | computeRotationAmounts(); | |||||
1925 | ||||||
1926 | if (!HasZeros) | |||||
1927 | return Select(N, false); | |||||
1928 | ||||||
1929 | // We currently have two techniques for handling results with zeros: early | |||||
1930 | // masking (the default) and late masking. Late masking is sometimes more | |||||
1931 | // efficient, but because the structure of the bit groups is different, it | |||||
1932 | // is hard to tell without generating both and comparing the results. With | |||||
1933 | // late masking, we ignore zeros in the resulting value when inserting each | |||||
1934 | // set of bit groups, and then mask in the zeros at the end. With early | |||||
1935 | // masking, we only insert the non-zero parts of the result at every step. | |||||
1936 | ||||||
1937 | unsigned InstCnt, InstCntLateMask; | |||||
1938 | DEBUG(dbgs() << "\tEarly masking:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tEarly masking:\n"; } } while (0); | |||||
1939 | SDNode *RN = Select(N, false, &InstCnt); | |||||
1940 | DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\tisel would use " << InstCnt << " instructions\n"; } } while (0); | |||||
1941 | ||||||
1942 | DEBUG(dbgs() << "\tLate masking:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tLate masking:\n"; } } while (0); | |||||
1943 | SDNode *RNLM = Select(N, true, &InstCntLateMask); | |||||
1944 | DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\tisel would use " << InstCntLateMask << " instructions\n"; } } while (0) | |||||
| ||||||
1945 | " instructions\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\t\tisel would use " << InstCntLateMask << " instructions\n"; } } while (0); | |||||
1946 | ||||||
1947 | if (InstCnt <= InstCntLateMask) { | |||||
1948 | DEBUG(dbgs() << "\tUsing early-masking for isel\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tUsing early-masking for isel\n" ; } } while (0); | |||||
1949 | return RN; | |||||
1950 | } | |||||
1951 | ||||||
1952 | DEBUG(dbgs() << "\tUsing late-masking for isel\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\tUsing late-masking for isel\n" ; } } while (0); | |||||
1953 | return RNLM; | |||||
1954 | } | |||||
1955 | }; | |||||
1956 | } // anonymous namespace | |||||
1957 | ||||||
1958 | SDNode *PPCDAGToDAGISel::SelectBitPermutation(SDNode *N) { | |||||
1959 | if (N->getValueType(0) != MVT::i32 && | |||||
1960 | N->getValueType(0) != MVT::i64) | |||||
1961 | return nullptr; | |||||
1962 | ||||||
1963 | if (!UseBitPermRewriter) | |||||
| ||||||
1964 | return nullptr; | |||||
1965 | ||||||
1966 | switch (N->getOpcode()) { | |||||
1967 | default: break; | |||||
1968 | case ISD::ROTL: | |||||
1969 | case ISD::SHL: | |||||
1970 | case ISD::SRL: | |||||
1971 | case ISD::AND: | |||||
1972 | case ISD::OR: { | |||||
1973 | BitPermutationSelector BPS(CurDAG); | |||||
1974 | return BPS.Select(N); | |||||
1975 | } | |||||
1976 | } | |||||
1977 | ||||||
1978 | return nullptr; | |||||
1979 | } | |||||
1980 | ||||||
1981 | /// SelectCC - Select a comparison of the specified values with the specified | |||||
1982 | /// condition code, returning the CR# of the expression. | |||||
1983 | SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, | |||||
1984 | ISD::CondCode CC, SDLoc dl) { | |||||
1985 | // Always select the LHS. | |||||
1986 | unsigned Opc; | |||||
1987 | ||||||
1988 | if (LHS.getValueType() == MVT::i32) { | |||||
1989 | unsigned Imm; | |||||
1990 | if (CC == ISD::SETEQ || CC == ISD::SETNE) { | |||||
1991 | if (isInt32Immediate(RHS, Imm)) { | |||||
1992 | // SETEQ/SETNE comparison with 16-bit immediate, fold it. | |||||
1993 | if (isUInt<16>(Imm)) | |||||
1994 | return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, | |||||
1995 | getI32Imm(Imm & 0xFFFF, dl)), | |||||
1996 | 0); | |||||
1997 | // If this is a 16-bit signed immediate, fold it. | |||||
1998 | if (isInt<16>((int)Imm)) | |||||
1999 | return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, | |||||
2000 | getI32Imm(Imm & 0xFFFF, dl)), | |||||
2001 | 0); | |||||
2002 | ||||||
2003 | // For non-equality comparisons, the default code would materialize the | |||||
2004 | // constant, then compare against it, like this: | |||||
2005 | // lis r2, 4660 | |||||
2006 | // ori r2, r2, 22136 | |||||
2007 | // cmpw cr0, r3, r2 | |||||
2008 | // Since we are just comparing for equality, we can emit this instead: | |||||
2009 | // xoris r0,r3,0x1234 | |||||
2010 | // cmplwi cr0,r0,0x5678 | |||||
2011 | // beq cr0,L6 | |||||
2012 | SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS, | |||||
2013 | getI32Imm(Imm >> 16, dl)), 0); | |||||
2014 | return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor, | |||||
2015 | getI32Imm(Imm & 0xFFFF, dl)), 0); | |||||
2016 | } | |||||
2017 | Opc = PPC::CMPLW; | |||||
2018 | } else if (ISD::isUnsignedIntSetCC(CC)) { | |||||
2019 | if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm)) | |||||
2020 | return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, | |||||
2021 | getI32Imm(Imm & 0xFFFF, dl)), 0); | |||||
2022 | Opc = PPC::CMPLW; | |||||
2023 | } else { | |||||
2024 | short SImm; | |||||
2025 | if (isIntS16Immediate(RHS, SImm)) | |||||
2026 | return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, | |||||
2027 | getI32Imm((int)SImm & 0xFFFF, | |||||
2028 | dl)), | |||||
2029 | 0); | |||||
2030 | Opc = PPC::CMPW; | |||||
2031 | } | |||||
2032 | } else if (LHS.getValueType() == MVT::i64) { | |||||
2033 | uint64_t Imm; | |||||
2034 | if (CC == ISD::SETEQ || CC == ISD::SETNE) { | |||||
2035 | if (isInt64Immediate(RHS.getNode(), Imm)) { | |||||
2036 | // SETEQ/SETNE comparison with 16-bit immediate, fold it. | |||||
2037 | if (isUInt<16>(Imm)) | |||||
2038 | return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, | |||||
2039 | getI32Imm(Imm & 0xFFFF, dl)), | |||||
2040 | 0); | |||||
2041 | // If this is a 16-bit signed immediate, fold it. | |||||
2042 | if (isInt<16>(Imm)) | |||||
2043 | return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, | |||||
2044 | getI32Imm(Imm & 0xFFFF, dl)), | |||||
2045 | 0); | |||||
2046 | ||||||
2047 | // For non-equality comparisons, the default code would materialize the | |||||
2048 | // constant, then compare against it, like this: | |||||
2049 | // lis r2, 4660 | |||||
2050 | // ori r2, r2, 22136 | |||||
2051 | // cmpd cr0, r3, r2 | |||||
2052 | // Since we are just comparing for equality, we can emit this instead: | |||||
2053 | // xoris r0,r3,0x1234 | |||||
2054 | // cmpldi cr0,r0,0x5678 | |||||
2055 | // beq cr0,L6 | |||||
2056 | if (isUInt<32>(Imm)) { | |||||
2057 | SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS, | |||||
2058 | getI64Imm(Imm >> 16, dl)), 0); | |||||
2059 | return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor, | |||||
2060 | getI64Imm(Imm & 0xFFFF, dl)), | |||||
2061 | 0); | |||||
2062 | } | |||||
2063 | } | |||||
2064 | Opc = PPC::CMPLD; | |||||
2065 | } else if (ISD::isUnsignedIntSetCC(CC)) { | |||||
2066 | if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm)) | |||||
2067 | return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, | |||||
2068 | getI64Imm(Imm & 0xFFFF, dl)), 0); | |||||
2069 | Opc = PPC::CMPLD; | |||||
2070 | } else { | |||||
2071 | short SImm; | |||||
2072 | if (isIntS16Immediate(RHS, SImm)) | |||||
2073 | return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, | |||||
2074 | getI64Imm(SImm & 0xFFFF, dl)), | |||||
2075 | 0); | |||||
2076 | Opc = PPC::CMPD; | |||||
2077 | } | |||||
2078 | } else if (LHS.getValueType() == MVT::f32) { | |||||
2079 | Opc = PPC::FCMPUS; | |||||
2080 | } else { | |||||
2081 | assert(LHS.getValueType() == MVT::f64 && "Unknown vt!")((LHS.getValueType() == MVT::f64 && "Unknown vt!") ? static_cast <void> (0) : __assert_fail ("LHS.getValueType() == MVT::f64 && \"Unknown vt!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2081, __PRETTY_FUNCTION__)); | |||||
2082 | Opc = PPCSubTarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD; | |||||
2083 | } | |||||
2084 | return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0); | |||||
2085 | } | |||||
2086 | ||||||
2087 | static PPC::Predicate getPredicateForSetCC(ISD::CondCode CC) { | |||||
2088 | switch (CC) { | |||||
2089 | case ISD::SETUEQ: | |||||
2090 | case ISD::SETONE: | |||||
2091 | case ISD::SETOLE: | |||||
2092 | case ISD::SETOGE: | |||||
2093 | llvm_unreachable("Should be lowered by legalize!")::llvm::llvm_unreachable_internal("Should be lowered by legalize!" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2093); | |||||
2094 | default: llvm_unreachable("Unknown condition!")::llvm::llvm_unreachable_internal("Unknown condition!", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2094); | |||||
2095 | case ISD::SETOEQ: | |||||
2096 | case ISD::SETEQ: return PPC::PRED_EQ; | |||||
2097 | case ISD::SETUNE: | |||||
2098 | case ISD::SETNE: return PPC::PRED_NE; | |||||
2099 | case ISD::SETOLT: | |||||
2100 | case ISD::SETLT: return PPC::PRED_LT; | |||||
2101 | case ISD::SETULE: | |||||
2102 | case ISD::SETLE: return PPC::PRED_LE; | |||||
2103 | case ISD::SETOGT: | |||||
2104 | case ISD::SETGT: return PPC::PRED_GT; | |||||
2105 | case ISD::SETUGE: | |||||
2106 | case ISD::SETGE: return PPC::PRED_GE; | |||||
2107 | case ISD::SETO: return PPC::PRED_NU; | |||||
2108 | case ISD::SETUO: return PPC::PRED_UN; | |||||
2109 | // These two are invalid for floating point. Assume we have int. | |||||
2110 | case ISD::SETULT: return PPC::PRED_LT; | |||||
2111 | case ISD::SETUGT: return PPC::PRED_GT; | |||||
2112 | } | |||||
2113 | } | |||||
2114 | ||||||
2115 | /// getCRIdxForSetCC - Return the index of the condition register field | |||||
2116 | /// associated with the SetCC condition, and whether or not the field is | |||||
2117 | /// treated as inverted. That is, lt = 0; ge = 0 inverted. | |||||
2118 | static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) { | |||||
2119 | Invert = false; | |||||
2120 | switch (CC) { | |||||
2121 | default: llvm_unreachable("Unknown condition!")::llvm::llvm_unreachable_internal("Unknown condition!", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2121); | |||||
2122 | case ISD::SETOLT: | |||||
2123 | case ISD::SETLT: return 0; // Bit #0 = SETOLT | |||||
2124 | case ISD::SETOGT: | |||||
2125 | case ISD::SETGT: return 1; // Bit #1 = SETOGT | |||||
2126 | case ISD::SETOEQ: | |||||
2127 | case ISD::SETEQ: return 2; // Bit #2 = SETOEQ | |||||
2128 | case ISD::SETUO: return 3; // Bit #3 = SETUO | |||||
2129 | case ISD::SETUGE: | |||||
2130 | case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE | |||||
2131 | case ISD::SETULE: | |||||
2132 | case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE | |||||
2133 | case ISD::SETUNE: | |||||
2134 | case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE | |||||
2135 | case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO | |||||
2136 | case ISD::SETUEQ: | |||||
2137 | case ISD::SETOGE: | |||||
2138 | case ISD::SETOLE: | |||||
2139 | case ISD::SETONE: | |||||
2140 | llvm_unreachable("Invalid branch code: should be expanded by legalize")::llvm::llvm_unreachable_internal("Invalid branch code: should be expanded by legalize" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2140); | |||||
2141 | // These are invalid for floating point. Assume integer. | |||||
2142 | case ISD::SETULT: return 0; | |||||
2143 | case ISD::SETUGT: return 1; | |||||
2144 | } | |||||
2145 | } | |||||
2146 | ||||||
2147 | // getVCmpInst: return the vector compare instruction for the specified | |||||
2148 | // vector type and condition code. Since this is for altivec specific code, | |||||
2149 | // only support the altivec types (v16i8, v8i16, v4i32, v2i64, and v4f32). | |||||
2150 | static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC, | |||||
2151 | bool HasVSX, bool &Swap, bool &Negate) { | |||||
2152 | Swap = false; | |||||
2153 | Negate = false; | |||||
2154 | ||||||
2155 | if (VecVT.isFloatingPoint()) { | |||||
2156 | /* Handle some cases by swapping input operands. */ | |||||
2157 | switch (CC) { | |||||
2158 | case ISD::SETLE: CC = ISD::SETGE; Swap = true; break; | |||||
2159 | case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; | |||||
2160 | case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break; | |||||
2161 | case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break; | |||||
2162 | case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; | |||||
2163 | case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break; | |||||
2164 | default: break; | |||||
2165 | } | |||||
2166 | /* Handle some cases by negating the result. */ | |||||
2167 | switch (CC) { | |||||
2168 | case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; | |||||
2169 | case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break; | |||||
2170 | case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break; | |||||
2171 | case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break; | |||||
2172 | default: break; | |||||
2173 | } | |||||
2174 | /* We have instructions implementing the remaining cases. */ | |||||
2175 | switch (CC) { | |||||
2176 | case ISD::SETEQ: | |||||
2177 | case ISD::SETOEQ: | |||||
2178 | if (VecVT == MVT::v4f32) | |||||
2179 | return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP; | |||||
2180 | else if (VecVT == MVT::v2f64) | |||||
2181 | return PPC::XVCMPEQDP; | |||||
2182 | break; | |||||
2183 | case ISD::SETGT: | |||||
2184 | case ISD::SETOGT: | |||||
2185 | if (VecVT == MVT::v4f32) | |||||
2186 | return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP; | |||||
2187 | else if (VecVT == MVT::v2f64) | |||||
2188 | return PPC::XVCMPGTDP; | |||||
2189 | break; | |||||
2190 | case ISD::SETGE: | |||||
2191 | case ISD::SETOGE: | |||||
2192 | if (VecVT == MVT::v4f32) | |||||
2193 | return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP; | |||||
2194 | else if (VecVT == MVT::v2f64) | |||||
2195 | return PPC::XVCMPGEDP; | |||||
2196 | break; | |||||
2197 | default: | |||||
2198 | break; | |||||
2199 | } | |||||
2200 | llvm_unreachable("Invalid floating-point vector compare condition")::llvm::llvm_unreachable_internal("Invalid floating-point vector compare condition" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2200); | |||||
2201 | } else { | |||||
2202 | /* Handle some cases by swapping input operands. */ | |||||
2203 | switch (CC) { | |||||
2204 | case ISD::SETGE: CC = ISD::SETLE; Swap = true; break; | |||||
2205 | case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; | |||||
2206 | case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; | |||||
2207 | case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break; | |||||
2208 | default: break; | |||||
2209 | } | |||||
2210 | /* Handle some cases by negating the result. */ | |||||
2211 | switch (CC) { | |||||
2212 | case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; | |||||
2213 | case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break; | |||||
2214 | case ISD::SETLE: CC = ISD::SETGT; Negate = true; break; | |||||
2215 | case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break; | |||||
2216 | default: break; | |||||
2217 | } | |||||
2218 | /* We have instructions implementing the remaining cases. */ | |||||
2219 | switch (CC) { | |||||
2220 | case ISD::SETEQ: | |||||
2221 | case ISD::SETUEQ: | |||||
2222 | if (VecVT == MVT::v16i8) | |||||
2223 | return PPC::VCMPEQUB; | |||||
2224 | else if (VecVT == MVT::v8i16) | |||||
2225 | return PPC::VCMPEQUH; | |||||
2226 | else if (VecVT == MVT::v4i32) | |||||
2227 | return PPC::VCMPEQUW; | |||||
2228 | else if (VecVT == MVT::v2i64) | |||||
2229 | return PPC::VCMPEQUD; | |||||
2230 | break; | |||||
2231 | case ISD::SETGT: | |||||
2232 | if (VecVT == MVT::v16i8) | |||||
2233 | return PPC::VCMPGTSB; | |||||
2234 | else if (VecVT == MVT::v8i16) | |||||
2235 | return PPC::VCMPGTSH; | |||||
2236 | else if (VecVT == MVT::v4i32) | |||||
2237 | return PPC::VCMPGTSW; | |||||
2238 | else if (VecVT == MVT::v2i64) | |||||
2239 | return PPC::VCMPGTSD; | |||||
2240 | break; | |||||
2241 | case ISD::SETUGT: | |||||
2242 | if (VecVT == MVT::v16i8) | |||||
2243 | return PPC::VCMPGTUB; | |||||
2244 | else if (VecVT == MVT::v8i16) | |||||
2245 | return PPC::VCMPGTUH; | |||||
2246 | else if (VecVT == MVT::v4i32) | |||||
2247 | return PPC::VCMPGTUW; | |||||
2248 | else if (VecVT == MVT::v2i64) | |||||
2249 | return PPC::VCMPGTUD; | |||||
2250 | break; | |||||
2251 | default: | |||||
2252 | break; | |||||
2253 | } | |||||
2254 | llvm_unreachable("Invalid integer vector compare condition")::llvm::llvm_unreachable_internal("Invalid integer vector compare condition" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2254); | |||||
2255 | } | |||||
2256 | } | |||||
2257 | ||||||
2258 | SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) { | |||||
2259 | SDLoc dl(N); | |||||
2260 | unsigned Imm; | |||||
2261 | ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); | |||||
2262 | EVT PtrVT = | |||||
2263 | CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); | |||||
2264 | bool isPPC64 = (PtrVT == MVT::i64); | |||||
2265 | ||||||
2266 | if (!PPCSubTarget->useCRBits() && | |||||
2267 | isInt32Immediate(N->getOperand(1), Imm)) { | |||||
2268 | // We can codegen setcc op, imm very efficiently compared to a brcond. | |||||
2269 | // Check for those cases here. | |||||
2270 | // setcc op, 0 | |||||
2271 | if (Imm == 0) { | |||||
2272 | SDValue Op = N->getOperand(0); | |||||
2273 | switch (CC) { | |||||
2274 | default: break; | |||||
2275 | case ISD::SETEQ: { | |||||
2276 | Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0); | |||||
2277 | SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl), | |||||
2278 | getI32Imm(31, dl) }; | |||||
2279 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2280 | } | |||||
2281 | case ISD::SETNE: { | |||||
2282 | if (isPPC64) break; | |||||
2283 | SDValue AD = | |||||
2284 | SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, | |||||
2285 | Op, getI32Imm(~0U, dl)), 0); | |||||
2286 | return CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, | |||||
2287 | AD.getValue(1)); | |||||
2288 | } | |||||
2289 | case ISD::SETLT: { | |||||
2290 | SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), | |||||
2291 | getI32Imm(31, dl) }; | |||||
2292 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2293 | } | |||||
2294 | case ISD::SETGT: { | |||||
2295 | SDValue T = | |||||
2296 | SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0); | |||||
2297 | T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0); | |||||
2298 | SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl), | |||||
2299 | getI32Imm(31, dl) }; | |||||
2300 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2301 | } | |||||
2302 | } | |||||
2303 | } else if (Imm == ~0U) { // setcc op, -1 | |||||
2304 | SDValue Op = N->getOperand(0); | |||||
2305 | switch (CC) { | |||||
2306 | default: break; | |||||
2307 | case ISD::SETEQ: | |||||
2308 | if (isPPC64) break; | |||||
2309 | Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, | |||||
2310 | Op, getI32Imm(1, dl)), 0); | |||||
2311 | return CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, | |||||
2312 | SDValue(CurDAG->getMachineNode(PPC::LI, dl, | |||||
2313 | MVT::i32, | |||||
2314 | getI32Imm(0, dl)), | |||||
2315 | 0), Op.getValue(1)); | |||||
2316 | case ISD::SETNE: { | |||||
2317 | if (isPPC64) break; | |||||
2318 | Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0); | |||||
2319 | SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, | |||||
2320 | Op, getI32Imm(~0U, dl)); | |||||
2321 | return CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), | |||||
2322 | Op, SDValue(AD, 1)); | |||||
2323 | } | |||||
2324 | case ISD::SETLT: { | |||||
2325 | SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op, | |||||
2326 | getI32Imm(1, dl)), 0); | |||||
2327 | SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD, | |||||
2328 | Op), 0); | |||||
2329 | SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl), | |||||
2330 | getI32Imm(31, dl) }; | |||||
2331 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2332 | } | |||||
2333 | case ISD::SETGT: { | |||||
2334 | SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), | |||||
2335 | getI32Imm(31, dl) }; | |||||
2336 | Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); | |||||
2337 | return CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, | |||||
2338 | getI32Imm(1, dl)); | |||||
2339 | } | |||||
2340 | } | |||||
2341 | } | |||||
2342 | } | |||||
2343 | ||||||
2344 | SDValue LHS = N->getOperand(0); | |||||
2345 | SDValue RHS = N->getOperand(1); | |||||
2346 | ||||||
2347 | // Altivec Vector compare instructions do not set any CR register by default and | |||||
2348 | // vector compare operations return the same type as the operands. | |||||
2349 | if (LHS.getValueType().isVector()) { | |||||
2350 | if (PPCSubTarget->hasQPX()) | |||||
2351 | return nullptr; | |||||
2352 | ||||||
2353 | EVT VecVT = LHS.getValueType(); | |||||
2354 | bool Swap, Negate; | |||||
2355 | unsigned int VCmpInst = getVCmpInst(VecVT.getSimpleVT(), CC, | |||||
2356 | PPCSubTarget->hasVSX(), Swap, Negate); | |||||
2357 | if (Swap) | |||||
2358 | std::swap(LHS, RHS); | |||||
2359 | ||||||
2360 | EVT ResVT = VecVT.changeVectorElementTypeToInteger(); | |||||
2361 | if (Negate) { | |||||
2362 | SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0); | |||||
2363 | return CurDAG->SelectNodeTo(N, PPCSubTarget->hasVSX() ? PPC::XXLNOR : | |||||
2364 | PPC::VNOR, | |||||
2365 | ResVT, VCmp, VCmp); | |||||
2366 | } | |||||
2367 | ||||||
2368 | return CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS); | |||||
2369 | } | |||||
2370 | ||||||
2371 | if (PPCSubTarget->useCRBits()) | |||||
2372 | return nullptr; | |||||
2373 | ||||||
2374 | bool Inv; | |||||
2375 | unsigned Idx = getCRIdxForSetCC(CC, Inv); | |||||
2376 | SDValue CCReg = SelectCC(LHS, RHS, CC, dl); | |||||
2377 | SDValue IntCR; | |||||
2378 | ||||||
2379 | // Force the ccreg into CR7. | |||||
2380 | SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32); | |||||
2381 | ||||||
2382 | SDValue InFlag(nullptr, 0); // Null incoming flag value. | |||||
2383 | CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg, | |||||
2384 | InFlag).getValue(1); | |||||
2385 | ||||||
2386 | IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg, | |||||
2387 | CCReg), 0); | |||||
2388 | ||||||
2389 | SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl), | |||||
2390 | getI32Imm(31, dl), getI32Imm(31, dl) }; | |||||
2391 | if (!Inv) | |||||
2392 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2393 | ||||||
2394 | // Get the specified bit. | |||||
2395 | SDValue Tmp = | |||||
2396 | SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); | |||||
2397 | return CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl)); | |||||
2398 | } | |||||
2399 | ||||||
2400 | SDNode *PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { | |||||
2401 | // Transfer memoperands. | |||||
2402 | MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); | |||||
2403 | MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); | |||||
2404 | cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1); | |||||
2405 | return Result; | |||||
2406 | } | |||||
2407 | ||||||
2408 | ||||||
2409 | // Select - Convert the specified operand from a target-independent to a | |||||
2410 | // target-specific node if it hasn't already been changed. | |||||
2411 | SDNode *PPCDAGToDAGISel::Select(SDNode *N) { | |||||
2412 | SDLoc dl(N); | |||||
2413 | if (N->isMachineOpcode()) { | |||||
2414 | N->setNodeId(-1); | |||||
2415 | return nullptr; // Already selected. | |||||
2416 | } | |||||
2417 | ||||||
2418 | // In case any misguided DAG-level optimizations form an ADD with a | |||||
2419 | // TargetConstant operand, crash here instead of miscompiling (by selecting | |||||
2420 | // an r+r add instead of some kind of r+i add). | |||||
2421 | if (N->getOpcode() == ISD::ADD && | |||||
2422 | N->getOperand(1).getOpcode() == ISD::TargetConstant) | |||||
2423 | llvm_unreachable("Invalid ADD with TargetConstant operand")::llvm::llvm_unreachable_internal("Invalid ADD with TargetConstant operand" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2423); | |||||
2424 | ||||||
2425 | // Try matching complex bit permutations before doing anything else. | |||||
2426 | if (SDNode *NN = SelectBitPermutation(N)) | |||||
2427 | return NN; | |||||
2428 | ||||||
2429 | switch (N->getOpcode()) { | |||||
2430 | default: break; | |||||
2431 | ||||||
2432 | case ISD::Constant: { | |||||
2433 | if (N->getValueType(0) == MVT::i64) | |||||
2434 | return SelectInt64(CurDAG, N); | |||||
2435 | break; | |||||
2436 | } | |||||
2437 | ||||||
2438 | case ISD::SETCC: { | |||||
2439 | SDNode *SN = SelectSETCC(N); | |||||
2440 | if (SN) | |||||
2441 | return SN; | |||||
2442 | break; | |||||
2443 | } | |||||
2444 | case PPCISD::GlobalBaseReg: | |||||
2445 | return getGlobalBaseReg(); | |||||
2446 | ||||||
2447 | case ISD::FrameIndex: | |||||
2448 | return getFrameIndex(N, N); | |||||
2449 | ||||||
2450 | case PPCISD::MFOCRF: { | |||||
2451 | SDValue InFlag = N->getOperand(1); | |||||
2452 | return CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, | |||||
2453 | N->getOperand(0), InFlag); | |||||
2454 | } | |||||
2455 | ||||||
2456 | case PPCISD::READ_TIME_BASE: { | |||||
2457 | return CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32, | |||||
2458 | MVT::Other, N->getOperand(0)); | |||||
2459 | } | |||||
2460 | ||||||
2461 | case PPCISD::SRA_ADDZE: { | |||||
2462 | SDValue N0 = N->getOperand(0); | |||||
2463 | SDValue ShiftAmt = | |||||
2464 | CurDAG->getTargetConstant(*cast<ConstantSDNode>(N->getOperand(1))-> | |||||
2465 | getConstantIntValue(), dl, | |||||
2466 | N->getValueType(0)); | |||||
2467 | if (N->getValueType(0) == MVT::i64) { | |||||
2468 | SDNode *Op = | |||||
2469 | CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue, | |||||
2470 | N0, ShiftAmt); | |||||
2471 | return CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, | |||||
2472 | SDValue(Op, 0), SDValue(Op, 1)); | |||||
2473 | } else { | |||||
2474 | assert(N->getValueType(0) == MVT::i32 &&((N->getValueType(0) == MVT::i32 && "Expecting i64 or i32 in PPCISD::SRA_ADDZE" ) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && \"Expecting i64 or i32 in PPCISD::SRA_ADDZE\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2475, __PRETTY_FUNCTION__)) | |||||
2475 | "Expecting i64 or i32 in PPCISD::SRA_ADDZE")((N->getValueType(0) == MVT::i32 && "Expecting i64 or i32 in PPCISD::SRA_ADDZE" ) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && \"Expecting i64 or i32 in PPCISD::SRA_ADDZE\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2475, __PRETTY_FUNCTION__)); | |||||
2476 | SDNode *Op = | |||||
2477 | CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue, | |||||
2478 | N0, ShiftAmt); | |||||
2479 | return CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, | |||||
2480 | SDValue(Op, 0), SDValue(Op, 1)); | |||||
2481 | } | |||||
2482 | } | |||||
2483 | ||||||
2484 | case ISD::LOAD: { | |||||
2485 | // Handle preincrement loads. | |||||
2486 | LoadSDNode *LD = cast<LoadSDNode>(N); | |||||
2487 | EVT LoadedVT = LD->getMemoryVT(); | |||||
2488 | ||||||
2489 | // Normal loads are handled by code generated from the .td file. | |||||
2490 | if (LD->getAddressingMode() != ISD::PRE_INC) | |||||
2491 | break; | |||||
2492 | ||||||
2493 | SDValue Offset = LD->getOffset(); | |||||
2494 | if (Offset.getOpcode() == ISD::TargetConstant || | |||||
2495 | Offset.getOpcode() == ISD::TargetGlobalAddress) { | |||||
2496 | ||||||
2497 | unsigned Opcode; | |||||
2498 | bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; | |||||
2499 | if (LD->getValueType(0) != MVT::i64) { | |||||
2500 | // Handle PPC32 integer and normal FP loads. | |||||
2501 | assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load" ) ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16) && \"Invalid sext update load\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2501, __PRETTY_FUNCTION__)); | |||||
2502 | switch (LoadedVT.getSimpleVT().SimpleTy) { | |||||
2503 | default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2503); | |||||
2504 | case MVT::f64: Opcode = PPC::LFDU; break; | |||||
2505 | case MVT::f32: Opcode = PPC::LFSU; break; | |||||
2506 | case MVT::i32: Opcode = PPC::LWZU; break; | |||||
2507 | case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break; | |||||
2508 | case MVT::i1: | |||||
2509 | case MVT::i8: Opcode = PPC::LBZU; break; | |||||
2510 | } | |||||
2511 | } else { | |||||
2512 | assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!")((LD->getValueType(0) == MVT::i64 && "Unknown load result type!" ) ? static_cast<void> (0) : __assert_fail ("LD->getValueType(0) == MVT::i64 && \"Unknown load result type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2512, __PRETTY_FUNCTION__)); | |||||
2513 | assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load" ) ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16) && \"Invalid sext update load\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2513, __PRETTY_FUNCTION__)); | |||||
2514 | switch (LoadedVT.getSimpleVT().SimpleTy) { | |||||
2515 | default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2515); | |||||
2516 | case MVT::i64: Opcode = PPC::LDU; break; | |||||
2517 | case MVT::i32: Opcode = PPC::LWZU8; break; | |||||
2518 | case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break; | |||||
2519 | case MVT::i1: | |||||
2520 | case MVT::i8: Opcode = PPC::LBZU8; break; | |||||
2521 | } | |||||
2522 | } | |||||
2523 | ||||||
2524 | SDValue Chain = LD->getChain(); | |||||
2525 | SDValue Base = LD->getBasePtr(); | |||||
2526 | SDValue Ops[] = { Offset, Base, Chain }; | |||||
2527 | return transferMemOperands( | |||||
2528 | N, CurDAG->getMachineNode( | |||||
2529 | Opcode, dl, LD->getValueType(0), | |||||
2530 | PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, | |||||
2531 | Ops)); | |||||
2532 | } else { | |||||
2533 | unsigned Opcode; | |||||
2534 | bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; | |||||
2535 | if (LD->getValueType(0) != MVT::i64) { | |||||
2536 | // Handle PPC32 integer and normal FP loads. | |||||
2537 | assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load" ) ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16) && \"Invalid sext update load\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2537, __PRETTY_FUNCTION__)); | |||||
2538 | switch (LoadedVT.getSimpleVT().SimpleTy) { | |||||
2539 | default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2539); | |||||
2540 | case MVT::v4f64: Opcode = PPC::QVLFDUX; break; // QPX | |||||
2541 | case MVT::v4f32: Opcode = PPC::QVLFSUX; break; // QPX | |||||
2542 | case MVT::f64: Opcode = PPC::LFDUX; break; | |||||
2543 | case MVT::f32: Opcode = PPC::LFSUX; break; | |||||
2544 | case MVT::i32: Opcode = PPC::LWZUX; break; | |||||
2545 | case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break; | |||||
2546 | case MVT::i1: | |||||
2547 | case MVT::i8: Opcode = PPC::LBZUX; break; | |||||
2548 | } | |||||
2549 | } else { | |||||
2550 | assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!")((LD->getValueType(0) == MVT::i64 && "Unknown load result type!" ) ? static_cast<void> (0) : __assert_fail ("LD->getValueType(0) == MVT::i64 && \"Unknown load result type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2550, __PRETTY_FUNCTION__)); | |||||
2551 | assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) &&(((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && "Invalid sext update load") ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && \"Invalid sext update load\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2552, __PRETTY_FUNCTION__)) | |||||
2552 | "Invalid sext update load")(((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && "Invalid sext update load") ? static_cast<void> (0) : __assert_fail ("(!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && \"Invalid sext update load\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2552, __PRETTY_FUNCTION__)); | |||||
2553 | switch (LoadedVT.getSimpleVT().SimpleTy) { | |||||
2554 | default: llvm_unreachable("Invalid PPC load type!")::llvm::llvm_unreachable_internal("Invalid PPC load type!", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2554); | |||||
2555 | case MVT::i64: Opcode = PPC::LDUX; break; | |||||
2556 | case MVT::i32: Opcode = isSExt ? PPC::LWAUX : PPC::LWZUX8; break; | |||||
2557 | case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break; | |||||
2558 | case MVT::i1: | |||||
2559 | case MVT::i8: Opcode = PPC::LBZUX8; break; | |||||
2560 | } | |||||
2561 | } | |||||
2562 | ||||||
2563 | SDValue Chain = LD->getChain(); | |||||
2564 | SDValue Base = LD->getBasePtr(); | |||||
2565 | SDValue Ops[] = { Base, Offset, Chain }; | |||||
2566 | return transferMemOperands( | |||||
2567 | N, CurDAG->getMachineNode( | |||||
2568 | Opcode, dl, LD->getValueType(0), | |||||
2569 | PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, | |||||
2570 | Ops)); | |||||
2571 | } | |||||
2572 | } | |||||
2573 | ||||||
2574 | case ISD::AND: { | |||||
2575 | unsigned Imm, Imm2, SH, MB, ME; | |||||
2576 | uint64_t Imm64; | |||||
2577 | ||||||
2578 | // If this is an and of a value rotated between 0 and 31 bits and then and'd | |||||
2579 | // with a mask, emit rlwinm | |||||
2580 | if (isInt32Immediate(N->getOperand(1), Imm) && | |||||
2581 | isRotateAndMask(N->getOperand(0).getNode(), Imm, false, SH, MB, ME)) { | |||||
2582 | SDValue Val = N->getOperand(0).getOperand(0); | |||||
2583 | SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl), | |||||
2584 | getI32Imm(ME, dl) }; | |||||
2585 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2586 | } | |||||
2587 | // If this is just a masked value where the input is not handled above, and | |||||
2588 | // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm | |||||
2589 | if (isInt32Immediate(N->getOperand(1), Imm) && | |||||
2590 | isRunOfOnes(Imm, MB, ME) && | |||||
2591 | N->getOperand(0).getOpcode() != ISD::ROTL) { | |||||
2592 | SDValue Val = N->getOperand(0); | |||||
2593 | SDValue Ops[] = { Val, getI32Imm(0, dl), getI32Imm(MB, dl), | |||||
2594 | getI32Imm(ME, dl) }; | |||||
2595 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2596 | } | |||||
2597 | // If this is a 64-bit zero-extension mask, emit rldicl. | |||||
2598 | if (isInt64Immediate(N->getOperand(1).getNode(), Imm64) && | |||||
2599 | isMask_64(Imm64)) { | |||||
2600 | SDValue Val = N->getOperand(0); | |||||
2601 | MB = 64 - countTrailingOnes(Imm64); | |||||
2602 | SH = 0; | |||||
2603 | ||||||
2604 | // If the operand is a logical right shift, we can fold it into this | |||||
2605 | // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb) | |||||
2606 | // for n <= mb. The right shift is really a left rotate followed by a | |||||
2607 | // mask, and this mask is a more-restrictive sub-mask of the mask implied | |||||
2608 | // by the shift. | |||||
2609 | if (Val.getOpcode() == ISD::SRL && | |||||
2610 | isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) { | |||||
2611 | assert(Imm < 64 && "Illegal shift amount")((Imm < 64 && "Illegal shift amount") ? static_cast <void> (0) : __assert_fail ("Imm < 64 && \"Illegal shift amount\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2611, __PRETTY_FUNCTION__)); | |||||
2612 | Val = Val.getOperand(0); | |||||
2613 | SH = 64 - Imm; | |||||
2614 | } | |||||
2615 | ||||||
2616 | SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) }; | |||||
2617 | return CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); | |||||
2618 | } | |||||
2619 | // AND X, 0 -> 0, not "rlwinm 32". | |||||
2620 | if (isInt32Immediate(N->getOperand(1), Imm) && (Imm == 0)) { | |||||
2621 | ReplaceUses(SDValue(N, 0), N->getOperand(1)); | |||||
2622 | return nullptr; | |||||
2623 | } | |||||
2624 | // ISD::OR doesn't get all the bitfield insertion fun. | |||||
2625 | // (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a | |||||
2626 | // bitfield insert. | |||||
2627 | if (isInt32Immediate(N->getOperand(1), Imm) && | |||||
2628 | N->getOperand(0).getOpcode() == ISD::OR && | |||||
2629 | isInt32Immediate(N->getOperand(0).getOperand(1), Imm2)) { | |||||
2630 | // The idea here is to check whether this is equivalent to: | |||||
2631 | // (c1 & m) | (x & ~m) | |||||
2632 | // where m is a run-of-ones mask. The logic here is that, for each bit in | |||||
2633 | // c1 and c2: | |||||
2634 | // - if both are 1, then the output will be 1. | |||||
2635 | // - if both are 0, then the output will be 0. | |||||
2636 | // - if the bit in c1 is 0, and the bit in c2 is 1, then the output will | |||||
2637 | // come from x. | |||||
2638 | // - if the bit in c1 is 1, and the bit in c2 is 0, then the output will | |||||
2639 | // be 0. | |||||
2640 | // If that last condition is never the case, then we can form m from the | |||||
2641 | // bits that are the same between c1 and c2. | |||||
2642 | unsigned MB, ME; | |||||
2643 | if (isRunOfOnes(~(Imm^Imm2), MB, ME) && !(~Imm & Imm2)) { | |||||
2644 | SDValue Ops[] = { N->getOperand(0).getOperand(0), | |||||
2645 | N->getOperand(0).getOperand(1), | |||||
2646 | getI32Imm(0, dl), getI32Imm(MB, dl), | |||||
2647 | getI32Imm(ME, dl) }; | |||||
2648 | return CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops); | |||||
2649 | } | |||||
2650 | } | |||||
2651 | ||||||
2652 | // Other cases are autogenerated. | |||||
2653 | break; | |||||
2654 | } | |||||
2655 | case ISD::OR: { | |||||
2656 | if (N->getValueType(0) == MVT::i32) | |||||
2657 | if (SDNode *I = SelectBitfieldInsert(N)) | |||||
2658 | return I; | |||||
2659 | ||||||
2660 | short Imm; | |||||
2661 | if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && | |||||
2662 | isIntS16Immediate(N->getOperand(1), Imm)) { | |||||
2663 | APInt LHSKnownZero, LHSKnownOne; | |||||
2664 | CurDAG->computeKnownBits(N->getOperand(0), LHSKnownZero, LHSKnownOne); | |||||
2665 | ||||||
2666 | // If this is equivalent to an add, then we can fold it with the | |||||
2667 | // FrameIndex calculation. | |||||
2668 | if ((LHSKnownZero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) | |||||
2669 | return getFrameIndex(N, N->getOperand(0).getNode(), (int)Imm); | |||||
2670 | } | |||||
2671 | ||||||
2672 | // Other cases are autogenerated. | |||||
2673 | break; | |||||
2674 | } | |||||
2675 | case ISD::ADD: { | |||||
2676 | short Imm; | |||||
2677 | if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && | |||||
2678 | isIntS16Immediate(N->getOperand(1), Imm)) | |||||
2679 | return getFrameIndex(N, N->getOperand(0).getNode(), (int)Imm); | |||||
2680 | ||||||
2681 | break; | |||||
2682 | } | |||||
2683 | case ISD::SHL: { | |||||
2684 | unsigned Imm, SH, MB, ME; | |||||
2685 | if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && | |||||
2686 | isRotateAndMask(N, Imm, true, SH, MB, ME)) { | |||||
2687 | SDValue Ops[] = { N->getOperand(0).getOperand(0), | |||||
2688 | getI32Imm(SH, dl), getI32Imm(MB, dl), | |||||
2689 | getI32Imm(ME, dl) }; | |||||
2690 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2691 | } | |||||
2692 | ||||||
2693 | // Other cases are autogenerated. | |||||
2694 | break; | |||||
2695 | } | |||||
2696 | case ISD::SRL: { | |||||
2697 | unsigned Imm, SH, MB, ME; | |||||
2698 | if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && | |||||
2699 | isRotateAndMask(N, Imm, true, SH, MB, ME)) { | |||||
2700 | SDValue Ops[] = { N->getOperand(0).getOperand(0), | |||||
2701 | getI32Imm(SH, dl), getI32Imm(MB, dl), | |||||
2702 | getI32Imm(ME, dl) }; | |||||
2703 | return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); | |||||
2704 | } | |||||
2705 | ||||||
2706 | // Other cases are autogenerated. | |||||
2707 | break; | |||||
2708 | } | |||||
2709 | // FIXME: Remove this once the ANDI glue bug is fixed: | |||||
2710 | case PPCISD::ANDIo_1_EQ_BIT: | |||||
2711 | case PPCISD::ANDIo_1_GT_BIT: { | |||||
2712 | if (!ANDIGlueBug) | |||||
2713 | break; | |||||
2714 | ||||||
2715 | EVT InVT = N->getOperand(0).getValueType(); | |||||
2716 | assert((InVT == MVT::i64 || InVT == MVT::i32) &&(((InVT == MVT::i64 || InVT == MVT::i32) && "Invalid input type for ANDIo_1_EQ_BIT" ) ? static_cast<void> (0) : __assert_fail ("(InVT == MVT::i64 || InVT == MVT::i32) && \"Invalid input type for ANDIo_1_EQ_BIT\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2717, __PRETTY_FUNCTION__)) | |||||
2717 | "Invalid input type for ANDIo_1_EQ_BIT")(((InVT == MVT::i64 || InVT == MVT::i32) && "Invalid input type for ANDIo_1_EQ_BIT" ) ? static_cast<void> (0) : __assert_fail ("(InVT == MVT::i64 || InVT == MVT::i32) && \"Invalid input type for ANDIo_1_EQ_BIT\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2717, __PRETTY_FUNCTION__)); | |||||
2718 | ||||||
2719 | unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDIo8 : PPC::ANDIo; | |||||
2720 | SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue, | |||||
2721 | N->getOperand(0), | |||||
2722 | CurDAG->getTargetConstant(1, dl, InVT)), | |||||
2723 | 0); | |||||
2724 | SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); | |||||
2725 | SDValue SRIdxVal = | |||||
2726 | CurDAG->getTargetConstant(N->getOpcode() == PPCISD::ANDIo_1_EQ_BIT ? | |||||
2727 | PPC::sub_eq : PPC::sub_gt, dl, MVT::i32); | |||||
2728 | ||||||
2729 | return CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, | |||||
2730 | CR0Reg, SRIdxVal, | |||||
2731 | SDValue(AndI.getNode(), 1) /* glue */); | |||||
2732 | } | |||||
2733 | case ISD::SELECT_CC: { | |||||
2734 | ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get(); | |||||
2735 | EVT PtrVT = | |||||
2736 | CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); | |||||
2737 | bool isPPC64 = (PtrVT == MVT::i64); | |||||
2738 | ||||||
2739 | // If this is a select of i1 operands, we'll pattern match it. | |||||
2740 | if (PPCSubTarget->useCRBits() && | |||||
2741 | N->getOperand(0).getValueType() == MVT::i1) | |||||
2742 | break; | |||||
2743 | ||||||
2744 | // Handle the setcc cases here. select_cc lhs, 0, 1, 0, cc | |||||
2745 | if (!isPPC64) | |||||
2746 | if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1))) | |||||
2747 | if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N->getOperand(2))) | |||||
2748 | if (ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N->getOperand(3))) | |||||
2749 | if (N1C->isNullValue() && N3C->isNullValue() && | |||||
2750 | N2C->getZExtValue() == 1ULL && CC == ISD::SETNE && | |||||
2751 | // FIXME: Implement this optzn for PPC64. | |||||
2752 | N->getValueType(0) == MVT::i32) { | |||||
2753 | SDNode *Tmp = | |||||
2754 | CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, | |||||
2755 | N->getOperand(0), getI32Imm(~0U, dl)); | |||||
2756 | return CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, | |||||
2757 | SDValue(Tmp, 0), N->getOperand(0), | |||||
2758 | SDValue(Tmp, 1)); | |||||
2759 | } | |||||
2760 | ||||||
2761 | SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl); | |||||
2762 | ||||||
2763 | if (N->getValueType(0) == MVT::i1) { | |||||
2764 | // An i1 select is: (c & t) | (!c & f). | |||||
2765 | bool Inv; | |||||
2766 | unsigned Idx = getCRIdxForSetCC(CC, Inv); | |||||
2767 | ||||||
2768 | unsigned SRI; | |||||
2769 | switch (Idx) { | |||||
2770 | default: llvm_unreachable("Invalid CC index")::llvm::llvm_unreachable_internal("Invalid CC index", "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2770); | |||||
2771 | case 0: SRI = PPC::sub_lt; break; | |||||
2772 | case 1: SRI = PPC::sub_gt; break; | |||||
2773 | case 2: SRI = PPC::sub_eq; break; | |||||
2774 | case 3: SRI = PPC::sub_un; break; | |||||
2775 | } | |||||
2776 | ||||||
2777 | SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg); | |||||
2778 | ||||||
2779 | SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1, | |||||
2780 | CCBit, CCBit), 0); | |||||
2781 | SDValue C = Inv ? NotCCBit : CCBit, | |||||
2782 | NotC = Inv ? CCBit : NotCCBit; | |||||
2783 | ||||||
2784 | SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, | |||||
2785 | C, N->getOperand(2)), 0); | |||||
2786 | SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, | |||||
2787 | NotC, N->getOperand(3)), 0); | |||||
2788 | ||||||
2789 | return CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF); | |||||
2790 | } | |||||
2791 | ||||||
2792 | unsigned BROpc = getPredicateForSetCC(CC); | |||||
2793 | ||||||
2794 | unsigned SelectCCOp; | |||||
2795 | if (N->getValueType(0) == MVT::i32) | |||||
2796 | SelectCCOp = PPC::SELECT_CC_I4; | |||||
2797 | else if (N->getValueType(0) == MVT::i64) | |||||
2798 | SelectCCOp = PPC::SELECT_CC_I8; | |||||
2799 | else if (N->getValueType(0) == MVT::f32) | |||||
2800 | if (PPCSubTarget->hasP8Vector()) | |||||
2801 | SelectCCOp = PPC::SELECT_CC_VSSRC; | |||||
2802 | else | |||||
2803 | SelectCCOp = PPC::SELECT_CC_F4; | |||||
2804 | else if (N->getValueType(0) == MVT::f64) | |||||
2805 | if (PPCSubTarget->hasVSX()) | |||||
2806 | SelectCCOp = PPC::SELECT_CC_VSFRC; | |||||
2807 | else | |||||
2808 | SelectCCOp = PPC::SELECT_CC_F8; | |||||
2809 | else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f64) | |||||
2810 | SelectCCOp = PPC::SELECT_CC_QFRC; | |||||
2811 | else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f32) | |||||
2812 | SelectCCOp = PPC::SELECT_CC_QSRC; | |||||
2813 | else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4i1) | |||||
2814 | SelectCCOp = PPC::SELECT_CC_QBRC; | |||||
2815 | else if (N->getValueType(0) == MVT::v2f64 || | |||||
2816 | N->getValueType(0) == MVT::v2i64) | |||||
2817 | SelectCCOp = PPC::SELECT_CC_VSRC; | |||||
2818 | else | |||||
2819 | SelectCCOp = PPC::SELECT_CC_VRRC; | |||||
2820 | ||||||
2821 | SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3), | |||||
2822 | getI32Imm(BROpc, dl) }; | |||||
2823 | return CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops); | |||||
2824 | } | |||||
2825 | case ISD::VSELECT: | |||||
2826 | if (PPCSubTarget->hasVSX()) { | |||||
2827 | SDValue Ops[] = { N->getOperand(2), N->getOperand(1), N->getOperand(0) }; | |||||
2828 | return CurDAG->SelectNodeTo(N, PPC::XXSEL, N->getValueType(0), Ops); | |||||
2829 | } | |||||
2830 | ||||||
2831 | break; | |||||
2832 | case ISD::VECTOR_SHUFFLE: | |||||
2833 | if (PPCSubTarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 || | |||||
2834 | N->getValueType(0) == MVT::v2i64)) { | |||||
2835 | ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); | |||||
2836 | ||||||
2837 | SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1), | |||||
2838 | Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1); | |||||
2839 | unsigned DM[2]; | |||||
2840 | ||||||
2841 | for (int i = 0; i < 2; ++i) | |||||
2842 | if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2) | |||||
2843 | DM[i] = 0; | |||||
2844 | else | |||||
2845 | DM[i] = 1; | |||||
2846 | ||||||
2847 | if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 && | |||||
2848 | Op1.getOpcode() == ISD::SCALAR_TO_VECTOR && | |||||
2849 | isa<LoadSDNode>(Op1.getOperand(0))) { | |||||
2850 | LoadSDNode *LD = cast<LoadSDNode>(Op1.getOperand(0)); | |||||
2851 | SDValue Base, Offset; | |||||
2852 | ||||||
2853 | if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() && | |||||
2854 | (LD->getMemoryVT() == MVT::f64 || | |||||
2855 | LD->getMemoryVT() == MVT::i64) && | |||||
2856 | SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { | |||||
2857 | SDValue Chain = LD->getChain(); | |||||
2858 | SDValue Ops[] = { Base, Offset, Chain }; | |||||
2859 | return CurDAG->SelectNodeTo(N, PPC::LXVDSX, | |||||
2860 | N->getValueType(0), Ops); | |||||
2861 | } | |||||
2862 | } | |||||
2863 | ||||||
2864 | // For little endian, we must swap the input operands and adjust | |||||
2865 | // the mask elements (reverse and invert them). | |||||
2866 | if (PPCSubTarget->isLittleEndian()) { | |||||
2867 | std::swap(Op1, Op2); | |||||
2868 | unsigned tmp = DM[0]; | |||||
2869 | DM[0] = 1 - DM[1]; | |||||
2870 | DM[1] = 1 - tmp; | |||||
2871 | } | |||||
2872 | ||||||
2873 | SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl, | |||||
2874 | MVT::i32); | |||||
2875 | SDValue Ops[] = { Op1, Op2, DMV }; | |||||
2876 | return CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops); | |||||
2877 | } | |||||
2878 | ||||||
2879 | break; | |||||
2880 | case PPCISD::BDNZ: | |||||
2881 | case PPCISD::BDZ: { | |||||
2882 | bool IsPPC64 = PPCSubTarget->isPPC64(); | |||||
2883 | SDValue Ops[] = { N->getOperand(1), N->getOperand(0) }; | |||||
2884 | return CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ ? | |||||
2885 | (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : | |||||
2886 | (IsPPC64 ? PPC::BDZ8 : PPC::BDZ), | |||||
2887 | MVT::Other, Ops); | |||||
2888 | } | |||||
2889 | case PPCISD::COND_BRANCH: { | |||||
2890 | // Op #0 is the Chain. | |||||
2891 | // Op #1 is the PPC::PRED_* number. | |||||
2892 | // Op #2 is the CR# | |||||
2893 | // Op #3 is the Dest MBB | |||||
2894 | // Op #4 is the Flag. | |||||
2895 | // Prevent PPC::PRED_* from being selected into LI. | |||||
2896 | unsigned PCC = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); | |||||
2897 | if (EnableBranchHint) | |||||
2898 | PCC |= getBranchHint(PCC, FuncInfo, N->getOperand(3)); | |||||
2899 | ||||||
2900 | SDValue Pred = getI32Imm(PCC, dl); | |||||
2901 | SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3), | |||||
2902 | N->getOperand(0), N->getOperand(4) }; | |||||
2903 | return CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); | |||||
2904 | } | |||||
2905 | case ISD::BR_CC: { | |||||
2906 | ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); | |||||
2907 | unsigned PCC = getPredicateForSetCC(CC); | |||||
2908 | ||||||
2909 | if (N->getOperand(2).getValueType() == MVT::i1) { | |||||
2910 | unsigned Opc; | |||||
2911 | bool Swap; | |||||
2912 | switch (PCC) { | |||||
2913 | default: llvm_unreachable("Unexpected Boolean-operand predicate")::llvm::llvm_unreachable_internal("Unexpected Boolean-operand predicate" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2913); | |||||
2914 | case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true; break; | |||||
2915 | case PPC::PRED_LE: Opc = PPC::CRORC; Swap = true; break; | |||||
2916 | case PPC::PRED_EQ: Opc = PPC::CREQV; Swap = false; break; | |||||
2917 | case PPC::PRED_GE: Opc = PPC::CRORC; Swap = false; break; | |||||
2918 | case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break; | |||||
2919 | case PPC::PRED_NE: Opc = PPC::CRXOR; Swap = false; break; | |||||
2920 | } | |||||
2921 | ||||||
2922 | SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1, | |||||
2923 | N->getOperand(Swap ? 3 : 2), | |||||
2924 | N->getOperand(Swap ? 2 : 3)), 0); | |||||
2925 | return CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, | |||||
2926 | BitComp, N->getOperand(4), N->getOperand(0)); | |||||
2927 | } | |||||
2928 | ||||||
2929 | if (EnableBranchHint) | |||||
2930 | PCC |= getBranchHint(PCC, FuncInfo, N->getOperand(4)); | |||||
2931 | ||||||
2932 | SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl); | |||||
2933 | SDValue Ops[] = { getI32Imm(PCC, dl), CondCode, | |||||
2934 | N->getOperand(4), N->getOperand(0) }; | |||||
2935 | return CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); | |||||
2936 | } | |||||
2937 | case ISD::BRIND: { | |||||
2938 | // FIXME: Should custom lower this. | |||||
2939 | SDValue Chain = N->getOperand(0); | |||||
2940 | SDValue Target = N->getOperand(1); | |||||
2941 | unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8; | |||||
2942 | unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8; | |||||
2943 | Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target, | |||||
2944 | Chain), 0); | |||||
2945 | return CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain); | |||||
2946 | } | |||||
2947 | case PPCISD::TOC_ENTRY: { | |||||
2948 | assert ((PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()) &&(((PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI() ) && "Only supported for 64-bit ABI and 32-bit SVR4") ? static_cast<void> (0) : __assert_fail ("(PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()) && \"Only supported for 64-bit ABI and 32-bit SVR4\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2949, __PRETTY_FUNCTION__)) | |||||
2949 | "Only supported for 64-bit ABI and 32-bit SVR4")(((PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI() ) && "Only supported for 64-bit ABI and 32-bit SVR4") ? static_cast<void> (0) : __assert_fail ("(PPCSubTarget->isPPC64() || PPCSubTarget->isSVR4ABI()) && \"Only supported for 64-bit ABI and 32-bit SVR4\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2949, __PRETTY_FUNCTION__)); | |||||
2950 | if (PPCSubTarget->isSVR4ABI() && !PPCSubTarget->isPPC64()) { | |||||
2951 | SDValue GA = N->getOperand(0); | |||||
2952 | return transferMemOperands(N, CurDAG->getMachineNode(PPC::LWZtoc, dl, | |||||
2953 | MVT::i32, GA, N->getOperand(1))); | |||||
2954 | } | |||||
2955 | ||||||
2956 | // For medium and large code model, we generate two instructions as | |||||
2957 | // described below. Otherwise we allow SelectCodeCommon to handle this, | |||||
2958 | // selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA. | |||||
2959 | CodeModel::Model CModel = TM.getCodeModel(); | |||||
2960 | if (CModel != CodeModel::Medium && CModel != CodeModel::Large) | |||||
2961 | break; | |||||
2962 | ||||||
2963 | // The first source operand is a TargetGlobalAddress or a TargetJumpTable. | |||||
2964 | // If it must be toc-referenced according to PPCSubTarget, we generate: | |||||
2965 | // LDtocL(<ga:@sym>, ADDIStocHA(%X2, <ga:@sym>)) | |||||
2966 | // Otherwise we generate: | |||||
2967 | // ADDItocL(ADDIStocHA(%X2, <ga:@sym>), <ga:@sym>) | |||||
2968 | SDValue GA = N->getOperand(0); | |||||
2969 | SDValue TOCbase = N->getOperand(1); | |||||
2970 | SDNode *Tmp = CurDAG->getMachineNode(PPC::ADDIStocHA, dl, MVT::i64, | |||||
2971 | TOCbase, GA); | |||||
2972 | ||||||
2973 | if (isa<JumpTableSDNode>(GA) || isa<BlockAddressSDNode>(GA) || | |||||
2974 | CModel == CodeModel::Large) | |||||
2975 | return transferMemOperands(N, CurDAG->getMachineNode(PPC::LDtocL, dl, | |||||
2976 | MVT::i64, GA, SDValue(Tmp, 0))); | |||||
2977 | ||||||
2978 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA)) { | |||||
2979 | const GlobalValue *GV = G->getGlobal(); | |||||
2980 | unsigned char GVFlags = PPCSubTarget->classifyGlobalReference(GV); | |||||
2981 | if (GVFlags & PPCII::MO_NLP_FLAG) { | |||||
2982 | return transferMemOperands(N, CurDAG->getMachineNode(PPC::LDtocL, dl, | |||||
2983 | MVT::i64, GA, SDValue(Tmp, 0))); | |||||
2984 | } | |||||
2985 | } | |||||
2986 | ||||||
2987 | return CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64, | |||||
2988 | SDValue(Tmp, 0), GA); | |||||
2989 | } | |||||
2990 | case PPCISD::PPC32_PICGOT: { | |||||
2991 | // Generate a PIC-safe GOT reference. | |||||
2992 | assert(!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI() &&((!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI () && "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4" ) ? static_cast<void> (0) : __assert_fail ("!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI() && \"PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2993, __PRETTY_FUNCTION__)) | |||||
2993 | "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4")((!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI () && "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4" ) ? static_cast<void> (0) : __assert_fail ("!PPCSubTarget->isPPC64() && PPCSubTarget->isSVR4ABI() && \"PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 2993, __PRETTY_FUNCTION__)); | |||||
2994 | return CurDAG->SelectNodeTo( | |||||
2995 | N, PPC::PPC32PICGOT, PPCLowering->getPointerTy(CurDAG->getDataLayout()), | |||||
2996 | MVT::i32); | |||||
2997 | } | |||||
2998 | case PPCISD::VADD_SPLAT: { | |||||
2999 | // This expands into one of three sequences, depending on whether | |||||
3000 | // the first operand is odd or even, positive or negative. | |||||
3001 | assert(isa<ConstantSDNode>(N->getOperand(0)) &&((isa<ConstantSDNode>(N->getOperand(0)) && isa <ConstantSDNode>(N->getOperand(1)) && "Invalid operand on VADD_SPLAT!" ) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(0)) && isa<ConstantSDNode>(N->getOperand(1)) && \"Invalid operand on VADD_SPLAT!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3003, __PRETTY_FUNCTION__)) | |||||
3002 | isa<ConstantSDNode>(N->getOperand(1)) &&((isa<ConstantSDNode>(N->getOperand(0)) && isa <ConstantSDNode>(N->getOperand(1)) && "Invalid operand on VADD_SPLAT!" ) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(0)) && isa<ConstantSDNode>(N->getOperand(1)) && \"Invalid operand on VADD_SPLAT!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3003, __PRETTY_FUNCTION__)) | |||||
3003 | "Invalid operand on VADD_SPLAT!")((isa<ConstantSDNode>(N->getOperand(0)) && isa <ConstantSDNode>(N->getOperand(1)) && "Invalid operand on VADD_SPLAT!" ) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(0)) && isa<ConstantSDNode>(N->getOperand(1)) && \"Invalid operand on VADD_SPLAT!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3003, __PRETTY_FUNCTION__)); | |||||
3004 | ||||||
3005 | int Elt = N->getConstantOperandVal(0); | |||||
3006 | int EltSize = N->getConstantOperandVal(1); | |||||
3007 | unsigned Opc1, Opc2, Opc3; | |||||
3008 | EVT VT; | |||||
3009 | ||||||
3010 | if (EltSize == 1) { | |||||
3011 | Opc1 = PPC::VSPLTISB; | |||||
3012 | Opc2 = PPC::VADDUBM; | |||||
3013 | Opc3 = PPC::VSUBUBM; | |||||
3014 | VT = MVT::v16i8; | |||||
3015 | } else if (EltSize == 2) { | |||||
3016 | Opc1 = PPC::VSPLTISH; | |||||
3017 | Opc2 = PPC::VADDUHM; | |||||
3018 | Opc3 = PPC::VSUBUHM; | |||||
3019 | VT = MVT::v8i16; | |||||
3020 | } else { | |||||
3021 | assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!")((EltSize == 4 && "Invalid element size on VADD_SPLAT!" ) ? static_cast<void> (0) : __assert_fail ("EltSize == 4 && \"Invalid element size on VADD_SPLAT!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3021, __PRETTY_FUNCTION__)); | |||||
3022 | Opc1 = PPC::VSPLTISW; | |||||
3023 | Opc2 = PPC::VADDUWM; | |||||
3024 | Opc3 = PPC::VSUBUWM; | |||||
3025 | VT = MVT::v4i32; | |||||
3026 | } | |||||
3027 | ||||||
3028 | if ((Elt & 1) == 0) { | |||||
3029 | // Elt is even, in the range [-32,-18] + [16,30]. | |||||
3030 | // | |||||
3031 | // Convert: VADD_SPLAT elt, size | |||||
3032 | // Into: tmp = VSPLTIS[BHW] elt | |||||
3033 | // VADDU[BHW]M tmp, tmp | |||||
3034 | // Where: [BHW] = B for size = 1, H for size = 2, W for size = 4 | |||||
3035 | SDValue EltVal = getI32Imm(Elt >> 1, dl); | |||||
3036 | SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); | |||||
3037 | SDValue TmpVal = SDValue(Tmp, 0); | |||||
3038 | return CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal); | |||||
3039 | ||||||
3040 | } else if (Elt > 0) { | |||||
3041 | // Elt is odd and positive, in the range [17,31]. | |||||
3042 | // | |||||
3043 | // Convert: VADD_SPLAT elt, size | |||||
3044 | // Into: tmp1 = VSPLTIS[BHW] elt-16 | |||||
3045 | // tmp2 = VSPLTIS[BHW] -16 | |||||
3046 | // VSUBU[BHW]M tmp1, tmp2 | |||||
3047 | SDValue EltVal = getI32Imm(Elt - 16, dl); | |||||
3048 | SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); | |||||
3049 | EltVal = getI32Imm(-16, dl); | |||||
3050 | SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); | |||||
3051 | return CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0), | |||||
3052 | SDValue(Tmp2, 0)); | |||||
3053 | ||||||
3054 | } else { | |||||
3055 | // Elt is odd and negative, in the range [-31,-17]. | |||||
3056 | // | |||||
3057 | // Convert: VADD_SPLAT elt, size | |||||
3058 | // Into: tmp1 = VSPLTIS[BHW] elt+16 | |||||
3059 | // tmp2 = VSPLTIS[BHW] -16 | |||||
3060 | // VADDU[BHW]M tmp1, tmp2 | |||||
3061 | SDValue EltVal = getI32Imm(Elt + 16, dl); | |||||
3062 | SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); | |||||
3063 | EltVal = getI32Imm(-16, dl); | |||||
3064 | SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); | |||||
3065 | return CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0), | |||||
3066 | SDValue(Tmp2, 0)); | |||||
3067 | } | |||||
3068 | } | |||||
3069 | } | |||||
3070 | ||||||
3071 | return SelectCode(N); | |||||
3072 | } | |||||
3073 | ||||||
3074 | // If the target supports the cmpb instruction, do the idiom recognition here. | |||||
3075 | // We don't do this as a DAG combine because we don't want to do it as nodes | |||||
3076 | // are being combined (because we might miss part of the eventual idiom). We | |||||
3077 | // don't want to do it during instruction selection because we want to reuse | |||||
3078 | // the logic for lowering the masking operations already part of the | |||||
3079 | // instruction selector. | |||||
3080 | SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) { | |||||
3081 | SDLoc dl(N); | |||||
3082 | ||||||
3083 | assert(N->getOpcode() == ISD::OR &&((N->getOpcode() == ISD::OR && "Only OR nodes are supported for CMPB" ) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Only OR nodes are supported for CMPB\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3084, __PRETTY_FUNCTION__)) | |||||
3084 | "Only OR nodes are supported for CMPB")((N->getOpcode() == ISD::OR && "Only OR nodes are supported for CMPB" ) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Only OR nodes are supported for CMPB\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3084, __PRETTY_FUNCTION__)); | |||||
3085 | ||||||
3086 | SDValue Res; | |||||
3087 | if (!PPCSubTarget->hasCMPB()) | |||||
3088 | return Res; | |||||
3089 | ||||||
3090 | if (N->getValueType(0) != MVT::i32 && | |||||
3091 | N->getValueType(0) != MVT::i64) | |||||
3092 | return Res; | |||||
3093 | ||||||
3094 | EVT VT = N->getValueType(0); | |||||
3095 | ||||||
3096 | SDValue RHS, LHS; | |||||
3097 | bool BytesFound[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; | |||||
3098 | uint64_t Mask = 0, Alt = 0; | |||||
3099 | ||||||
3100 | auto IsByteSelectCC = [this](SDValue O, unsigned &b, | |||||
3101 | uint64_t &Mask, uint64_t &Alt, | |||||
3102 | SDValue &LHS, SDValue &RHS) { | |||||
3103 | if (O.getOpcode() != ISD::SELECT_CC) | |||||
3104 | return false; | |||||
3105 | ISD::CondCode CC = cast<CondCodeSDNode>(O.getOperand(4))->get(); | |||||
3106 | ||||||
3107 | if (!isa<ConstantSDNode>(O.getOperand(2)) || | |||||
3108 | !isa<ConstantSDNode>(O.getOperand(3))) | |||||
3109 | return false; | |||||
3110 | ||||||
3111 | uint64_t PM = O.getConstantOperandVal(2); | |||||
3112 | uint64_t PAlt = O.getConstantOperandVal(3); | |||||
3113 | for (b = 0; b < 8; ++b) { | |||||
3114 | uint64_t Mask = UINT64_C(0xFF)0xFFUL << (8*b); | |||||
3115 | if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt) | |||||
3116 | break; | |||||
3117 | } | |||||
3118 | ||||||
3119 | if (b == 8) | |||||
3120 | return false; | |||||
3121 | Mask |= PM; | |||||
3122 | Alt |= PAlt; | |||||
3123 | ||||||
3124 | if (!isa<ConstantSDNode>(O.getOperand(1)) || | |||||
3125 | O.getConstantOperandVal(1) != 0) { | |||||
3126 | SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1); | |||||
3127 | if (Op0.getOpcode() == ISD::TRUNCATE) | |||||
3128 | Op0 = Op0.getOperand(0); | |||||
3129 | if (Op1.getOpcode() == ISD::TRUNCATE) | |||||
3130 | Op1 = Op1.getOperand(0); | |||||
3131 | ||||||
3132 | if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL && | |||||
3133 | Op0.getOperand(1) == Op1.getOperand(1) && CC == ISD::SETEQ && | |||||
3134 | isa<ConstantSDNode>(Op0.getOperand(1))) { | |||||
3135 | ||||||
3136 | unsigned Bits = Op0.getValueType().getSizeInBits(); | |||||
3137 | if (b != Bits/8-1) | |||||
3138 | return false; | |||||
3139 | if (Op0.getConstantOperandVal(1) != Bits-8) | |||||
3140 | return false; | |||||
3141 | ||||||
3142 | LHS = Op0.getOperand(0); | |||||
3143 | RHS = Op1.getOperand(0); | |||||
3144 | return true; | |||||
3145 | } | |||||
3146 | ||||||
3147 | // When we have small integers (i16 to be specific), the form present | |||||
3148 | // post-legalization uses SETULT in the SELECT_CC for the | |||||
3149 | // higher-order byte, depending on the fact that the | |||||
3150 | // even-higher-order bytes are known to all be zero, for example: | |||||
3151 | // select_cc (xor $lhs, $rhs), 256, 65280, 0, setult | |||||
3152 | // (so when the second byte is the same, because all higher-order | |||||
3153 | // bits from bytes 3 and 4 are known to be zero, the result of the | |||||
3154 | // xor can be at most 255) | |||||
3155 | if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT && | |||||
3156 | isa<ConstantSDNode>(O.getOperand(1))) { | |||||
3157 | ||||||
3158 | uint64_t ULim = O.getConstantOperandVal(1); | |||||
3159 | if (ULim != (UINT64_C(1)1UL << b*8)) | |||||
3160 | return false; | |||||
3161 | ||||||
3162 | // Now we need to make sure that the upper bytes are known to be | |||||
3163 | // zero. | |||||
3164 | unsigned Bits = Op0.getValueType().getSizeInBits(); | |||||
3165 | if (!CurDAG->MaskedValueIsZero(Op0, | |||||
3166 | APInt::getHighBitsSet(Bits, Bits - (b+1)*8))) | |||||
3167 | return false; | |||||
3168 | ||||||
3169 | LHS = Op0.getOperand(0); | |||||
3170 | RHS = Op0.getOperand(1); | |||||
3171 | return true; | |||||
3172 | } | |||||
3173 | ||||||
3174 | return false; | |||||
3175 | } | |||||
3176 | ||||||
3177 | if (CC != ISD::SETEQ) | |||||
3178 | return false; | |||||
3179 | ||||||
3180 | SDValue Op = O.getOperand(0); | |||||
3181 | if (Op.getOpcode() == ISD::AND) { | |||||
3182 | if (!isa<ConstantSDNode>(Op.getOperand(1))) | |||||
3183 | return false; | |||||
3184 | if (Op.getConstantOperandVal(1) != (UINT64_C(0xFF)0xFFUL << (8*b))) | |||||
3185 | return false; | |||||
3186 | ||||||
3187 | SDValue XOR = Op.getOperand(0); | |||||
3188 | if (XOR.getOpcode() == ISD::TRUNCATE) | |||||
3189 | XOR = XOR.getOperand(0); | |||||
3190 | if (XOR.getOpcode() != ISD::XOR) | |||||
3191 | return false; | |||||
3192 | ||||||
3193 | LHS = XOR.getOperand(0); | |||||
3194 | RHS = XOR.getOperand(1); | |||||
3195 | return true; | |||||
3196 | } else if (Op.getOpcode() == ISD::SRL) { | |||||
3197 | if (!isa<ConstantSDNode>(Op.getOperand(1))) | |||||
3198 | return false; | |||||
3199 | unsigned Bits = Op.getValueType().getSizeInBits(); | |||||
3200 | if (b != Bits/8-1) | |||||
3201 | return false; | |||||
3202 | if (Op.getConstantOperandVal(1) != Bits-8) | |||||
3203 | return false; | |||||
3204 | ||||||
3205 | SDValue XOR = Op.getOperand(0); | |||||
3206 | if (XOR.getOpcode() == ISD::TRUNCATE) | |||||
3207 | XOR = XOR.getOperand(0); | |||||
3208 | if (XOR.getOpcode() != ISD::XOR) | |||||
3209 | return false; | |||||
3210 | ||||||
3211 | LHS = XOR.getOperand(0); | |||||
3212 | RHS = XOR.getOperand(1); | |||||
3213 | return true; | |||||
3214 | } | |||||
3215 | ||||||
3216 | return false; | |||||
3217 | }; | |||||
3218 | ||||||
3219 | SmallVector<SDValue, 8> Queue(1, SDValue(N, 0)); | |||||
3220 | while (!Queue.empty()) { | |||||
3221 | SDValue V = Queue.pop_back_val(); | |||||
3222 | ||||||
3223 | for (const SDValue &O : V.getNode()->ops()) { | |||||
3224 | unsigned b; | |||||
3225 | uint64_t M = 0, A = 0; | |||||
3226 | SDValue OLHS, ORHS; | |||||
3227 | if (O.getOpcode() == ISD::OR) { | |||||
3228 | Queue.push_back(O); | |||||
3229 | } else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) { | |||||
3230 | if (!LHS) { | |||||
3231 | LHS = OLHS; | |||||
3232 | RHS = ORHS; | |||||
3233 | BytesFound[b] = true; | |||||
3234 | Mask |= M; | |||||
3235 | Alt |= A; | |||||
3236 | } else if ((LHS == ORHS && RHS == OLHS) || | |||||
3237 | (RHS == ORHS && LHS == OLHS)) { | |||||
3238 | BytesFound[b] = true; | |||||
3239 | Mask |= M; | |||||
3240 | Alt |= A; | |||||
3241 | } else { | |||||
3242 | return Res; | |||||
3243 | } | |||||
3244 | } else { | |||||
3245 | return Res; | |||||
3246 | } | |||||
3247 | } | |||||
3248 | } | |||||
3249 | ||||||
3250 | unsigned LastB = 0, BCnt = 0; | |||||
3251 | for (unsigned i = 0; i < 8; ++i) | |||||
3252 | if (BytesFound[LastB]) { | |||||
3253 | ++BCnt; | |||||
3254 | LastB = i; | |||||
3255 | } | |||||
3256 | ||||||
3257 | if (!LastB || BCnt < 2) | |||||
3258 | return Res; | |||||
3259 | ||||||
3260 | // Because we'll be zero-extending the output anyway if don't have a specific | |||||
3261 | // value for each input byte (via the Mask), we can 'anyext' the inputs. | |||||
3262 | if (LHS.getValueType() != VT) { | |||||
3263 | LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT); | |||||
3264 | RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT); | |||||
3265 | } | |||||
3266 | ||||||
3267 | Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS); | |||||
3268 | ||||||
3269 | bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1)-1L; | |||||
3270 | if (NonTrivialMask && !Alt) { | |||||
3271 | // Res = Mask & CMPB | |||||
3272 | Res = CurDAG->getNode(ISD::AND, dl, VT, Res, | |||||
3273 | CurDAG->getConstant(Mask, dl, VT)); | |||||
3274 | } else if (Alt) { | |||||
3275 | // Res = (CMPB & Mask) | (~CMPB & Alt) | |||||
3276 | // Which, as suggested here: | |||||
3277 | // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge | |||||
3278 | // can be written as: | |||||
3279 | // Res = Alt ^ ((Alt ^ Mask) & CMPB) | |||||
3280 | // useful because the (Alt ^ Mask) can be pre-computed. | |||||
3281 | Res = CurDAG->getNode(ISD::AND, dl, VT, Res, | |||||
3282 | CurDAG->getConstant(Mask ^ Alt, dl, VT)); | |||||
3283 | Res = CurDAG->getNode(ISD::XOR, dl, VT, Res, | |||||
3284 | CurDAG->getConstant(Alt, dl, VT)); | |||||
3285 | } | |||||
3286 | ||||||
3287 | return Res; | |||||
3288 | } | |||||
3289 | ||||||
3290 | // When CR bit registers are enabled, an extension of an i1 variable to a i32 | |||||
3291 | // or i64 value is lowered in terms of a SELECT_I[48] operation, and thus | |||||
3292 | // involves constant materialization of a 0 or a 1 or both. If the result of | |||||
3293 | // the extension is then operated upon by some operator that can be constant | |||||
3294 | // folded with a constant 0 or 1, and that constant can be materialized using | |||||
3295 | // only one instruction (like a zero or one), then we should fold in those | |||||
3296 | // operations with the select. | |||||
3297 | void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) { | |||||
3298 | if (!PPCSubTarget->useCRBits()) | |||||
3299 | return; | |||||
3300 | ||||||
3301 | if (N->getOpcode() != ISD::ZERO_EXTEND && | |||||
3302 | N->getOpcode() != ISD::SIGN_EXTEND && | |||||
3303 | N->getOpcode() != ISD::ANY_EXTEND) | |||||
3304 | return; | |||||
3305 | ||||||
3306 | if (N->getOperand(0).getValueType() != MVT::i1) | |||||
3307 | return; | |||||
3308 | ||||||
3309 | if (!N->hasOneUse()) | |||||
3310 | return; | |||||
3311 | ||||||
3312 | SDLoc dl(N); | |||||
3313 | EVT VT = N->getValueType(0); | |||||
3314 | SDValue Cond = N->getOperand(0); | |||||
3315 | SDValue ConstTrue = | |||||
3316 | CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT); | |||||
3317 | SDValue ConstFalse = CurDAG->getConstant(0, dl, VT); | |||||
3318 | ||||||
3319 | do { | |||||
3320 | SDNode *User = *N->use_begin(); | |||||
3321 | if (User->getNumOperands() != 2) | |||||
3322 | break; | |||||
3323 | ||||||
3324 | auto TryFold = [this, N, User, dl](SDValue Val) { | |||||
3325 | SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1); | |||||
3326 | SDValue O0 = UserO0.getNode() == N ? Val : UserO0; | |||||
3327 | SDValue O1 = UserO1.getNode() == N ? Val : UserO1; | |||||
3328 | ||||||
3329 | return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl, | |||||
3330 | User->getValueType(0), | |||||
3331 | O0.getNode(), O1.getNode()); | |||||
3332 | }; | |||||
3333 | ||||||
3334 | SDValue TrueRes = TryFold(ConstTrue); | |||||
3335 | if (!TrueRes) | |||||
3336 | break; | |||||
3337 | SDValue FalseRes = TryFold(ConstFalse); | |||||
3338 | if (!FalseRes) | |||||
3339 | break; | |||||
3340 | ||||||
3341 | // For us to materialize these using one instruction, we must be able to | |||||
3342 | // represent them as signed 16-bit integers. | |||||
3343 | uint64_t True = cast<ConstantSDNode>(TrueRes)->getZExtValue(), | |||||
3344 | False = cast<ConstantSDNode>(FalseRes)->getZExtValue(); | |||||
3345 | if (!isInt<16>(True) || !isInt<16>(False)) | |||||
3346 | break; | |||||
3347 | ||||||
3348 | // We can replace User with a new SELECT node, and try again to see if we | |||||
3349 | // can fold the select with its user. | |||||
3350 | Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes); | |||||
3351 | N = User; | |||||
3352 | ConstTrue = TrueRes; | |||||
3353 | ConstFalse = FalseRes; | |||||
3354 | } while (N->hasOneUse()); | |||||
3355 | } | |||||
3356 | ||||||
3357 | void PPCDAGToDAGISel::PreprocessISelDAG() { | |||||
3358 | SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode()); | |||||
3359 | ++Position; | |||||
3360 | ||||||
3361 | bool MadeChange = false; | |||||
3362 | while (Position != CurDAG->allnodes_begin()) { | |||||
3363 | SDNode *N = &*--Position; | |||||
3364 | if (N->use_empty()) | |||||
3365 | continue; | |||||
3366 | ||||||
3367 | SDValue Res; | |||||
3368 | switch (N->getOpcode()) { | |||||
3369 | default: break; | |||||
3370 | case ISD::OR: | |||||
3371 | Res = combineToCMPB(N); | |||||
3372 | break; | |||||
3373 | } | |||||
3374 | ||||||
3375 | if (!Res) | |||||
3376 | foldBoolExts(Res, N); | |||||
3377 | ||||||
3378 | if (Res) { | |||||
3379 | DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "PPC DAG preprocessing replacing:\nOld: " ; } } while (0); | |||||
3380 | DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { N->dump(CurDAG); } } while (0); | |||||
3381 | DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\nNew: "; } } while (0); | |||||
3382 | DEBUG(Res.getNode()->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { Res.getNode()->dump(CurDAG); } } while ( 0); | |||||
3383 | DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\n"; } } while (0); | |||||
3384 | ||||||
3385 | CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); | |||||
3386 | MadeChange = true; | |||||
3387 | } | |||||
3388 | } | |||||
3389 | ||||||
3390 | if (MadeChange) | |||||
3391 | CurDAG->RemoveDeadNodes(); | |||||
3392 | } | |||||
3393 | ||||||
3394 | /// PostprocessISelDAG - Perform some late peephole optimizations | |||||
3395 | /// on the DAG representation. | |||||
3396 | void PPCDAGToDAGISel::PostprocessISelDAG() { | |||||
3397 | ||||||
3398 | // Skip peepholes at -O0. | |||||
3399 | if (TM.getOptLevel() == CodeGenOpt::None) | |||||
3400 | return; | |||||
3401 | ||||||
3402 | PeepholePPC64(); | |||||
3403 | PeepholeCROps(); | |||||
3404 | PeepholePPC64ZExt(); | |||||
3405 | } | |||||
3406 | ||||||
3407 | // Check if all users of this node will become isel where the second operand | |||||
3408 | // is the constant zero. If this is so, and if we can negate the condition, | |||||
3409 | // then we can flip the true and false operands. This will allow the zero to | |||||
3410 | // be folded with the isel so that we don't need to materialize a register | |||||
3411 | // containing zero. | |||||
3412 | bool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) { | |||||
3413 | // If we're not using isel, then this does not matter. | |||||
3414 | if (!PPCSubTarget->hasISEL()) | |||||
3415 | return false; | |||||
3416 | ||||||
3417 | for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); | |||||
3418 | UI != UE; ++UI) { | |||||
3419 | SDNode *User = *UI; | |||||
3420 | if (!User->isMachineOpcode()) | |||||
3421 | return false; | |||||
3422 | if (User->getMachineOpcode() != PPC::SELECT_I4 && | |||||
3423 | User->getMachineOpcode() != PPC::SELECT_I8) | |||||
3424 | return false; | |||||
3425 | ||||||
3426 | SDNode *Op2 = User->getOperand(2).getNode(); | |||||
3427 | if (!Op2->isMachineOpcode()) | |||||
3428 | return false; | |||||
3429 | ||||||
3430 | if (Op2->getMachineOpcode() != PPC::LI && | |||||
3431 | Op2->getMachineOpcode() != PPC::LI8) | |||||
3432 | return false; | |||||
3433 | ||||||
3434 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op2->getOperand(0)); | |||||
3435 | if (!C) | |||||
3436 | return false; | |||||
3437 | ||||||
3438 | if (!C->isNullValue()) | |||||
3439 | return false; | |||||
3440 | } | |||||
3441 | ||||||
3442 | return true; | |||||
3443 | } | |||||
3444 | ||||||
3445 | void PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) { | |||||
3446 | SmallVector<SDNode *, 4> ToReplace; | |||||
3447 | for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); | |||||
3448 | UI != UE; ++UI) { | |||||
3449 | SDNode *User = *UI; | |||||
3450 | assert((User->getMachineOpcode() == PPC::SELECT_I4 ||(((User->getMachineOpcode() == PPC::SELECT_I4 || User-> getMachineOpcode() == PPC::SELECT_I8) && "Must have all select users" ) ? static_cast<void> (0) : __assert_fail ("(User->getMachineOpcode() == PPC::SELECT_I4 || User->getMachineOpcode() == PPC::SELECT_I8) && \"Must have all select users\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3452, __PRETTY_FUNCTION__)) | |||||
3451 | User->getMachineOpcode() == PPC::SELECT_I8) &&(((User->getMachineOpcode() == PPC::SELECT_I4 || User-> getMachineOpcode() == PPC::SELECT_I8) && "Must have all select users" ) ? static_cast<void> (0) : __assert_fail ("(User->getMachineOpcode() == PPC::SELECT_I4 || User->getMachineOpcode() == PPC::SELECT_I8) && \"Must have all select users\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3452, __PRETTY_FUNCTION__)) | |||||
3452 | "Must have all select users")(((User->getMachineOpcode() == PPC::SELECT_I4 || User-> getMachineOpcode() == PPC::SELECT_I8) && "Must have all select users" ) ? static_cast<void> (0) : __assert_fail ("(User->getMachineOpcode() == PPC::SELECT_I4 || User->getMachineOpcode() == PPC::SELECT_I8) && \"Must have all select users\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 3452, __PRETTY_FUNCTION__)); | |||||
3453 | ToReplace.push_back(User); | |||||
3454 | } | |||||
3455 | ||||||
3456 | for (SmallVector<SDNode *, 4>::iterator UI = ToReplace.begin(), | |||||
3457 | UE = ToReplace.end(); UI != UE; ++UI) { | |||||
3458 | SDNode *User = *UI; | |||||
3459 | SDNode *ResNode = | |||||
3460 | CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User), | |||||
3461 | User->getValueType(0), User->getOperand(0), | |||||
3462 | User->getOperand(2), | |||||
3463 | User->getOperand(1)); | |||||
3464 | ||||||
3465 | DEBUG(dbgs() << "CR Peephole replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "CR Peephole replacing:\nOld: " ; } } while (0); | |||||
3466 | DEBUG(User->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { User->dump(CurDAG); } } while (0); | |||||
3467 | DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\nNew: "; } } while (0); | |||||
3468 | DEBUG(ResNode->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { ResNode->dump(CurDAG); } } while (0); | |||||
3469 | DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\n"; } } while (0); | |||||
3470 | ||||||
3471 | ReplaceUses(User, ResNode); | |||||
3472 | } | |||||
3473 | } | |||||
3474 | ||||||
3475 | void PPCDAGToDAGISel::PeepholeCROps() { | |||||
3476 | bool IsModified; | |||||
3477 | do { | |||||
3478 | IsModified = false; | |||||
3479 | for (SDNode &Node : CurDAG->allnodes()) { | |||||
3480 | MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node); | |||||
3481 | if (!MachineNode || MachineNode->use_empty()) | |||||
3482 | continue; | |||||
3483 | SDNode *ResNode = MachineNode; | |||||
3484 | ||||||
3485 | bool Op1Set = false, Op1Unset = false, | |||||
3486 | Op1Not = false, | |||||
3487 | Op2Set = false, Op2Unset = false, | |||||
3488 | Op2Not = false; | |||||
3489 | ||||||
3490 | unsigned Opcode = MachineNode->getMachineOpcode(); | |||||
3491 | switch (Opcode) { | |||||
3492 | default: break; | |||||
3493 | case PPC::CRAND: | |||||
3494 | case PPC::CRNAND: | |||||
3495 | case PPC::CROR: | |||||
3496 | case PPC::CRXOR: | |||||
3497 | case PPC::CRNOR: | |||||
3498 | case PPC::CREQV: | |||||
3499 | case PPC::CRANDC: | |||||
3500 | case PPC::CRORC: { | |||||
3501 | SDValue Op = MachineNode->getOperand(1); | |||||
3502 | if (Op.isMachineOpcode()) { | |||||
3503 | if (Op.getMachineOpcode() == PPC::CRSET) | |||||
3504 | Op2Set = true; | |||||
3505 | else if (Op.getMachineOpcode() == PPC::CRUNSET) | |||||
3506 | Op2Unset = true; | |||||
3507 | else if (Op.getMachineOpcode() == PPC::CRNOR && | |||||
3508 | Op.getOperand(0) == Op.getOperand(1)) | |||||
3509 | Op2Not = true; | |||||
3510 | } | |||||
3511 | } // fallthrough | |||||
3512 | case PPC::BC: | |||||
3513 | case PPC::BCn: | |||||
3514 | case PPC::SELECT_I4: | |||||
3515 | case PPC::SELECT_I8: | |||||
3516 | case PPC::SELECT_F4: | |||||
3517 | case PPC::SELECT_F8: | |||||
3518 | case PPC::SELECT_QFRC: | |||||
3519 | case PPC::SELECT_QSRC: | |||||
3520 | case PPC::SELECT_QBRC: | |||||
3521 | case PPC::SELECT_VRRC: | |||||
3522 | case PPC::SELECT_VSFRC: | |||||
3523 | case PPC::SELECT_VSSRC: | |||||
3524 | case PPC::SELECT_VSRC: { | |||||
3525 | SDValue Op = MachineNode->getOperand(0); | |||||
3526 | if (Op.isMachineOpcode()) { | |||||
3527 | if (Op.getMachineOpcode() == PPC::CRSET) | |||||
3528 | Op1Set = true; | |||||
3529 | else if (Op.getMachineOpcode() == PPC::CRUNSET) | |||||
3530 | Op1Unset = true; | |||||
3531 | else if (Op.getMachineOpcode() == PPC::CRNOR && | |||||
3532 | Op.getOperand(0) == Op.getOperand(1)) | |||||
3533 | Op1Not = true; | |||||
3534 | } | |||||
3535 | } | |||||
3536 | break; | |||||
3537 | } | |||||
3538 | ||||||
3539 | bool SelectSwap = false; | |||||
3540 | switch (Opcode) { | |||||
3541 | default: break; | |||||
3542 | case PPC::CRAND: | |||||
3543 | if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) | |||||
3544 | // x & x = x | |||||
3545 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3546 | else if (Op1Set) | |||||
3547 | // 1 & y = y | |||||
3548 | ResNode = MachineNode->getOperand(1).getNode(); | |||||
3549 | else if (Op2Set) | |||||
3550 | // x & 1 = x | |||||
3551 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3552 | else if (Op1Unset || Op2Unset) | |||||
3553 | // x & 0 = 0 & y = 0 | |||||
3554 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), | |||||
3555 | MVT::i1); | |||||
3556 | else if (Op1Not) | |||||
3557 | // ~x & y = andc(y, x) | |||||
3558 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), | |||||
3559 | MVT::i1, MachineNode->getOperand(1), | |||||
3560 | MachineNode->getOperand(0). | |||||
3561 | getOperand(0)); | |||||
3562 | else if (Op2Not) | |||||
3563 | // x & ~y = andc(x, y) | |||||
3564 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), | |||||
3565 | MVT::i1, MachineNode->getOperand(0), | |||||
3566 | MachineNode->getOperand(1). | |||||
3567 | getOperand(0)); | |||||
3568 | else if (AllUsersSelectZero(MachineNode)) | |||||
3569 | ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), | |||||
3570 | MVT::i1, MachineNode->getOperand(0), | |||||
3571 | MachineNode->getOperand(1)), | |||||
3572 | SelectSwap = true; | |||||
3573 | break; | |||||
3574 | case PPC::CRNAND: | |||||
3575 | if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) | |||||
3576 | // nand(x, x) -> nor(x, x) | |||||
3577 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3578 | MVT::i1, MachineNode->getOperand(0), | |||||
3579 | MachineNode->getOperand(0)); | |||||
3580 | else if (Op1Set) | |||||
3581 | // nand(1, y) -> nor(y, y) | |||||
3582 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3583 | MVT::i1, MachineNode->getOperand(1), | |||||
3584 | MachineNode->getOperand(1)); | |||||
3585 | else if (Op2Set) | |||||
3586 | // nand(x, 1) -> nor(x, x) | |||||
3587 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3588 | MVT::i1, MachineNode->getOperand(0), | |||||
3589 | MachineNode->getOperand(0)); | |||||
3590 | else if (Op1Unset || Op2Unset) | |||||
3591 | // nand(x, 0) = nand(0, y) = 1 | |||||
3592 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), | |||||
3593 | MVT::i1); | |||||
3594 | else if (Op1Not) | |||||
3595 | // nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y) | |||||
3596 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), | |||||
3597 | MVT::i1, MachineNode->getOperand(0). | |||||
3598 | getOperand(0), | |||||
3599 | MachineNode->getOperand(1)); | |||||
3600 | else if (Op2Not) | |||||
3601 | // nand(x, ~y) = ~x | y = orc(y, x) | |||||
3602 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), | |||||
3603 | MVT::i1, MachineNode->getOperand(1). | |||||
3604 | getOperand(0), | |||||
3605 | MachineNode->getOperand(0)); | |||||
3606 | else if (AllUsersSelectZero(MachineNode)) | |||||
3607 | ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), | |||||
3608 | MVT::i1, MachineNode->getOperand(0), | |||||
3609 | MachineNode->getOperand(1)), | |||||
3610 | SelectSwap = true; | |||||
3611 | break; | |||||
3612 | case PPC::CROR: | |||||
3613 | if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) | |||||
3614 | // x | x = x | |||||
3615 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3616 | else if (Op1Set || Op2Set) | |||||
3617 | // x | 1 = 1 | y = 1 | |||||
3618 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), | |||||
3619 | MVT::i1); | |||||
3620 | else if (Op1Unset) | |||||
3621 | // 0 | y = y | |||||
3622 | ResNode = MachineNode->getOperand(1).getNode(); | |||||
3623 | else if (Op2Unset) | |||||
3624 | // x | 0 = x | |||||
3625 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3626 | else if (Op1Not) | |||||
3627 | // ~x | y = orc(y, x) | |||||
3628 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), | |||||
3629 | MVT::i1, MachineNode->getOperand(1), | |||||
3630 | MachineNode->getOperand(0). | |||||
3631 | getOperand(0)); | |||||
3632 | else if (Op2Not) | |||||
3633 | // x | ~y = orc(x, y) | |||||
3634 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), | |||||
3635 | MVT::i1, MachineNode->getOperand(0), | |||||
3636 | MachineNode->getOperand(1). | |||||
3637 | getOperand(0)); | |||||
3638 | else if (AllUsersSelectZero(MachineNode)) | |||||
3639 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3640 | MVT::i1, MachineNode->getOperand(0), | |||||
3641 | MachineNode->getOperand(1)), | |||||
3642 | SelectSwap = true; | |||||
3643 | break; | |||||
3644 | case PPC::CRXOR: | |||||
3645 | if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) | |||||
3646 | // xor(x, x) = 0 | |||||
3647 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), | |||||
3648 | MVT::i1); | |||||
3649 | else if (Op1Set) | |||||
3650 | // xor(1, y) -> nor(y, y) | |||||
3651 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3652 | MVT::i1, MachineNode->getOperand(1), | |||||
3653 | MachineNode->getOperand(1)); | |||||
3654 | else if (Op2Set) | |||||
3655 | // xor(x, 1) -> nor(x, x) | |||||
3656 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3657 | MVT::i1, MachineNode->getOperand(0), | |||||
3658 | MachineNode->getOperand(0)); | |||||
3659 | else if (Op1Unset) | |||||
3660 | // xor(0, y) = y | |||||
3661 | ResNode = MachineNode->getOperand(1).getNode(); | |||||
3662 | else if (Op2Unset) | |||||
3663 | // xor(x, 0) = x | |||||
3664 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3665 | else if (Op1Not) | |||||
3666 | // xor(~x, y) = eqv(x, y) | |||||
3667 | ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), | |||||
3668 | MVT::i1, MachineNode->getOperand(0). | |||||
3669 | getOperand(0), | |||||
3670 | MachineNode->getOperand(1)); | |||||
3671 | else if (Op2Not) | |||||
3672 | // xor(x, ~y) = eqv(x, y) | |||||
3673 | ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), | |||||
3674 | MVT::i1, MachineNode->getOperand(0), | |||||
3675 | MachineNode->getOperand(1). | |||||
3676 | getOperand(0)); | |||||
3677 | else if (AllUsersSelectZero(MachineNode)) | |||||
3678 | ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), | |||||
3679 | MVT::i1, MachineNode->getOperand(0), | |||||
3680 | MachineNode->getOperand(1)), | |||||
3681 | SelectSwap = true; | |||||
3682 | break; | |||||
3683 | case PPC::CRNOR: | |||||
3684 | if (Op1Set || Op2Set) | |||||
3685 | // nor(1, y) -> 0 | |||||
3686 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), | |||||
3687 | MVT::i1); | |||||
3688 | else if (Op1Unset) | |||||
3689 | // nor(0, y) = ~y -> nor(y, y) | |||||
3690 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3691 | MVT::i1, MachineNode->getOperand(1), | |||||
3692 | MachineNode->getOperand(1)); | |||||
3693 | else if (Op2Unset) | |||||
3694 | // nor(x, 0) = ~x | |||||
3695 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3696 | MVT::i1, MachineNode->getOperand(0), | |||||
3697 | MachineNode->getOperand(0)); | |||||
3698 | else if (Op1Not) | |||||
3699 | // nor(~x, y) = andc(x, y) | |||||
3700 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), | |||||
3701 | MVT::i1, MachineNode->getOperand(0). | |||||
3702 | getOperand(0), | |||||
3703 | MachineNode->getOperand(1)); | |||||
3704 | else if (Op2Not) | |||||
3705 | // nor(x, ~y) = andc(y, x) | |||||
3706 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), | |||||
3707 | MVT::i1, MachineNode->getOperand(1). | |||||
3708 | getOperand(0), | |||||
3709 | MachineNode->getOperand(0)); | |||||
3710 | else if (AllUsersSelectZero(MachineNode)) | |||||
3711 | ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), | |||||
3712 | MVT::i1, MachineNode->getOperand(0), | |||||
3713 | MachineNode->getOperand(1)), | |||||
3714 | SelectSwap = true; | |||||
3715 | break; | |||||
3716 | case PPC::CREQV: | |||||
3717 | if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) | |||||
3718 | // eqv(x, x) = 1 | |||||
3719 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), | |||||
3720 | MVT::i1); | |||||
3721 | else if (Op1Set) | |||||
3722 | // eqv(1, y) = y | |||||
3723 | ResNode = MachineNode->getOperand(1).getNode(); | |||||
3724 | else if (Op2Set) | |||||
3725 | // eqv(x, 1) = x | |||||
3726 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3727 | else if (Op1Unset) | |||||
3728 | // eqv(0, y) = ~y -> nor(y, y) | |||||
3729 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3730 | MVT::i1, MachineNode->getOperand(1), | |||||
3731 | MachineNode->getOperand(1)); | |||||
3732 | else if (Op2Unset) | |||||
3733 | // eqv(x, 0) = ~x | |||||
3734 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3735 | MVT::i1, MachineNode->getOperand(0), | |||||
3736 | MachineNode->getOperand(0)); | |||||
3737 | else if (Op1Not) | |||||
3738 | // eqv(~x, y) = xor(x, y) | |||||
3739 | ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), | |||||
3740 | MVT::i1, MachineNode->getOperand(0). | |||||
3741 | getOperand(0), | |||||
3742 | MachineNode->getOperand(1)); | |||||
3743 | else if (Op2Not) | |||||
3744 | // eqv(x, ~y) = xor(x, y) | |||||
3745 | ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), | |||||
3746 | MVT::i1, MachineNode->getOperand(0), | |||||
3747 | MachineNode->getOperand(1). | |||||
3748 | getOperand(0)); | |||||
3749 | else if (AllUsersSelectZero(MachineNode)) | |||||
3750 | ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), | |||||
3751 | MVT::i1, MachineNode->getOperand(0), | |||||
3752 | MachineNode->getOperand(1)), | |||||
3753 | SelectSwap = true; | |||||
3754 | break; | |||||
3755 | case PPC::CRANDC: | |||||
3756 | if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) | |||||
3757 | // andc(x, x) = 0 | |||||
3758 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), | |||||
3759 | MVT::i1); | |||||
3760 | else if (Op1Set) | |||||
3761 | // andc(1, y) = ~y | |||||
3762 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3763 | MVT::i1, MachineNode->getOperand(1), | |||||
3764 | MachineNode->getOperand(1)); | |||||
3765 | else if (Op1Unset || Op2Set) | |||||
3766 | // andc(0, y) = andc(x, 1) = 0 | |||||
3767 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), | |||||
3768 | MVT::i1); | |||||
3769 | else if (Op2Unset) | |||||
3770 | // andc(x, 0) = x | |||||
3771 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3772 | else if (Op1Not) | |||||
3773 | // andc(~x, y) = ~(x | y) = nor(x, y) | |||||
3774 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3775 | MVT::i1, MachineNode->getOperand(0). | |||||
3776 | getOperand(0), | |||||
3777 | MachineNode->getOperand(1)); | |||||
3778 | else if (Op2Not) | |||||
3779 | // andc(x, ~y) = x & y | |||||
3780 | ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), | |||||
3781 | MVT::i1, MachineNode->getOperand(0), | |||||
3782 | MachineNode->getOperand(1). | |||||
3783 | getOperand(0)); | |||||
3784 | else if (AllUsersSelectZero(MachineNode)) | |||||
3785 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), | |||||
3786 | MVT::i1, MachineNode->getOperand(1), | |||||
3787 | MachineNode->getOperand(0)), | |||||
3788 | SelectSwap = true; | |||||
3789 | break; | |||||
3790 | case PPC::CRORC: | |||||
3791 | if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) | |||||
3792 | // orc(x, x) = 1 | |||||
3793 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), | |||||
3794 | MVT::i1); | |||||
3795 | else if (Op1Set || Op2Unset) | |||||
3796 | // orc(1, y) = orc(x, 0) = 1 | |||||
3797 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), | |||||
3798 | MVT::i1); | |||||
3799 | else if (Op2Set) | |||||
3800 | // orc(x, 1) = x | |||||
3801 | ResNode = MachineNode->getOperand(0).getNode(); | |||||
3802 | else if (Op1Unset) | |||||
3803 | // orc(0, y) = ~y | |||||
3804 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), | |||||
3805 | MVT::i1, MachineNode->getOperand(1), | |||||
3806 | MachineNode->getOperand(1)); | |||||
3807 | else if (Op1Not) | |||||
3808 | // orc(~x, y) = ~(x & y) = nand(x, y) | |||||
3809 | ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), | |||||
3810 | MVT::i1, MachineNode->getOperand(0). | |||||
3811 | getOperand(0), | |||||
3812 | MachineNode->getOperand(1)); | |||||
3813 | else if (Op2Not) | |||||
3814 | // orc(x, ~y) = x | y | |||||
3815 | ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), | |||||
3816 | MVT::i1, MachineNode->getOperand(0), | |||||
3817 | MachineNode->getOperand(1). | |||||
3818 | getOperand(0)); | |||||
3819 | else if (AllUsersSelectZero(MachineNode)) | |||||
3820 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), | |||||
3821 | MVT::i1, MachineNode->getOperand(1), | |||||
3822 | MachineNode->getOperand(0)), | |||||
3823 | SelectSwap = true; | |||||
3824 | break; | |||||
3825 | case PPC::SELECT_I4: | |||||
3826 | case PPC::SELECT_I8: | |||||
3827 | case PPC::SELECT_F4: | |||||
3828 | case PPC::SELECT_F8: | |||||
3829 | case PPC::SELECT_QFRC: | |||||
3830 | case PPC::SELECT_QSRC: | |||||
3831 | case PPC::SELECT_QBRC: | |||||
3832 | case PPC::SELECT_VRRC: | |||||
3833 | case PPC::SELECT_VSFRC: | |||||
3834 | case PPC::SELECT_VSSRC: | |||||
3835 | case PPC::SELECT_VSRC: | |||||
3836 | if (Op1Set) | |||||
3837 | ResNode = MachineNode->getOperand(1).getNode(); | |||||
3838 | else if (Op1Unset) | |||||
3839 | ResNode = MachineNode->getOperand(2).getNode(); | |||||
3840 | else if (Op1Not) | |||||
3841 | ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(), | |||||
3842 | SDLoc(MachineNode), | |||||
3843 | MachineNode->getValueType(0), | |||||
3844 | MachineNode->getOperand(0). | |||||
3845 | getOperand(0), | |||||
3846 | MachineNode->getOperand(2), | |||||
3847 | MachineNode->getOperand(1)); | |||||
3848 | break; | |||||
3849 | case PPC::BC: | |||||
3850 | case PPC::BCn: | |||||
3851 | if (Op1Not) | |||||
3852 | ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn : | |||||
3853 | PPC::BC, | |||||
3854 | SDLoc(MachineNode), | |||||
3855 | MVT::Other, | |||||
3856 | MachineNode->getOperand(0). | |||||
3857 | getOperand(0), | |||||
3858 | MachineNode->getOperand(1), | |||||
3859 | MachineNode->getOperand(2)); | |||||
3860 | // FIXME: Handle Op1Set, Op1Unset here too. | |||||
3861 | break; | |||||
3862 | } | |||||
3863 | ||||||
3864 | // If we're inverting this node because it is used only by selects that | |||||
3865 | // we'd like to swap, then swap the selects before the node replacement. | |||||
3866 | if (SelectSwap) | |||||
3867 | SwapAllSelectUsers(MachineNode); | |||||
3868 | ||||||
3869 | if (ResNode != MachineNode) { | |||||
3870 | DEBUG(dbgs() << "CR Peephole replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "CR Peephole replacing:\nOld: " ; } } while (0); | |||||
3871 | DEBUG(MachineNode->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { MachineNode->dump(CurDAG); } } while (0 ); | |||||
3872 | DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\nNew: "; } } while (0); | |||||
3873 | DEBUG(ResNode->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { ResNode->dump(CurDAG); } } while (0); | |||||
3874 | DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\n"; } } while (0); | |||||
3875 | ||||||
3876 | ReplaceUses(MachineNode, ResNode); | |||||
3877 | IsModified = true; | |||||
3878 | } | |||||
3879 | } | |||||
3880 | if (IsModified) | |||||
3881 | CurDAG->RemoveDeadNodes(); | |||||
3882 | } while (IsModified); | |||||
3883 | } | |||||
3884 | ||||||
3885 | // Gather the set of 32-bit operations that are known to have their | |||||
3886 | // higher-order 32 bits zero, where ToPromote contains all such operations. | |||||
3887 | static bool PeepholePPC64ZExtGather(SDValue Op32, | |||||
3888 | SmallPtrSetImpl<SDNode *> &ToPromote) { | |||||
3889 | if (!Op32.isMachineOpcode()) | |||||
3890 | return false; | |||||
3891 | ||||||
3892 | // First, check for the "frontier" instructions (those that will clear the | |||||
3893 | // higher-order 32 bits. | |||||
3894 | ||||||
3895 | // For RLWINM and RLWNM, we need to make sure that the mask does not wrap | |||||
3896 | // around. If it does not, then these instructions will clear the | |||||
3897 | // higher-order bits. | |||||
3898 | if ((Op32.getMachineOpcode() == PPC::RLWINM || | |||||
3899 | Op32.getMachineOpcode() == PPC::RLWNM) && | |||||
3900 | Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) { | |||||
3901 | ToPromote.insert(Op32.getNode()); | |||||
3902 | return true; | |||||
3903 | } | |||||
3904 | ||||||
3905 | // SLW and SRW always clear the higher-order bits. | |||||
3906 | if (Op32.getMachineOpcode() == PPC::SLW || | |||||
3907 | Op32.getMachineOpcode() == PPC::SRW) { | |||||
3908 | ToPromote.insert(Op32.getNode()); | |||||
3909 | return true; | |||||
3910 | } | |||||
3911 | ||||||
3912 | // For LI and LIS, we need the immediate to be positive (so that it is not | |||||
3913 | // sign extended). | |||||
3914 | if (Op32.getMachineOpcode() == PPC::LI || | |||||
3915 | Op32.getMachineOpcode() == PPC::LIS) { | |||||
3916 | if (!isUInt<15>(Op32.getConstantOperandVal(0))) | |||||
3917 | return false; | |||||
3918 | ||||||
3919 | ToPromote.insert(Op32.getNode()); | |||||
3920 | return true; | |||||
3921 | } | |||||
3922 | ||||||
3923 | // LHBRX and LWBRX always clear the higher-order bits. | |||||
3924 | if (Op32.getMachineOpcode() == PPC::LHBRX || | |||||
3925 | Op32.getMachineOpcode() == PPC::LWBRX) { | |||||
3926 | ToPromote.insert(Op32.getNode()); | |||||
3927 | return true; | |||||
3928 | } | |||||
3929 | ||||||
3930 | // CNTLZW always produces a 64-bit value in [0,32], and so is zero extended. | |||||
3931 | if (Op32.getMachineOpcode() == PPC::CNTLZW) { | |||||
3932 | ToPromote.insert(Op32.getNode()); | |||||
3933 | return true; | |||||
3934 | } | |||||
3935 | ||||||
3936 | // Next, check for those instructions we can look through. | |||||
3937 | ||||||
3938 | // Assuming the mask does not wrap around, then the higher-order bits are | |||||
3939 | // taken directly from the first operand. | |||||
3940 | if (Op32.getMachineOpcode() == PPC::RLWIMI && | |||||
3941 | Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) { | |||||
3942 | SmallPtrSet<SDNode *, 16> ToPromote1; | |||||
3943 | if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) | |||||
3944 | return false; | |||||
3945 | ||||||
3946 | ToPromote.insert(Op32.getNode()); | |||||
3947 | ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); | |||||
3948 | return true; | |||||
3949 | } | |||||
3950 | ||||||
3951 | // For OR, the higher-order bits are zero if that is true for both operands. | |||||
3952 | // For SELECT_I4, the same is true (but the relevant operand numbers are | |||||
3953 | // shifted by 1). | |||||
3954 | if (Op32.getMachineOpcode() == PPC::OR || | |||||
3955 | Op32.getMachineOpcode() == PPC::SELECT_I4) { | |||||
3956 | unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0; | |||||
3957 | SmallPtrSet<SDNode *, 16> ToPromote1; | |||||
3958 | if (!PeepholePPC64ZExtGather(Op32.getOperand(B+0), ToPromote1)) | |||||
3959 | return false; | |||||
3960 | if (!PeepholePPC64ZExtGather(Op32.getOperand(B+1), ToPromote1)) | |||||
3961 | return false; | |||||
3962 | ||||||
3963 | ToPromote.insert(Op32.getNode()); | |||||
3964 | ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); | |||||
3965 | return true; | |||||
3966 | } | |||||
3967 | ||||||
3968 | // For ORI and ORIS, we need the higher-order bits of the first operand to be | |||||
3969 | // zero, and also for the constant to be positive (so that it is not sign | |||||
3970 | // extended). | |||||
3971 | if (Op32.getMachineOpcode() == PPC::ORI || | |||||
3972 | Op32.getMachineOpcode() == PPC::ORIS) { | |||||
3973 | SmallPtrSet<SDNode *, 16> ToPromote1; | |||||
3974 | if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) | |||||
3975 | return false; | |||||
3976 | if (!isUInt<15>(Op32.getConstantOperandVal(1))) | |||||
3977 | return false; | |||||
3978 | ||||||
3979 | ToPromote.insert(Op32.getNode()); | |||||
3980 | ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); | |||||
3981 | return true; | |||||
3982 | } | |||||
3983 | ||||||
3984 | // The higher-order bits of AND are zero if that is true for at least one of | |||||
3985 | // the operands. | |||||
3986 | if (Op32.getMachineOpcode() == PPC::AND) { | |||||
3987 | SmallPtrSet<SDNode *, 16> ToPromote1, ToPromote2; | |||||
3988 | bool Op0OK = | |||||
3989 | PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); | |||||
3990 | bool Op1OK = | |||||
3991 | PeepholePPC64ZExtGather(Op32.getOperand(1), ToPromote2); | |||||
3992 | if (!Op0OK && !Op1OK) | |||||
3993 | return false; | |||||
3994 | ||||||
3995 | ToPromote.insert(Op32.getNode()); | |||||
3996 | ||||||
3997 | if (Op0OK) | |||||
3998 | ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); | |||||
3999 | ||||||
4000 | if (Op1OK) | |||||
4001 | ToPromote.insert(ToPromote2.begin(), ToPromote2.end()); | |||||
4002 | ||||||
4003 | return true; | |||||
4004 | } | |||||
4005 | ||||||
4006 | // For ANDI and ANDIS, the higher-order bits are zero if either that is true | |||||
4007 | // of the first operand, or if the second operand is positive (so that it is | |||||
4008 | // not sign extended). | |||||
4009 | if (Op32.getMachineOpcode() == PPC::ANDIo || | |||||
4010 | Op32.getMachineOpcode() == PPC::ANDISo) { | |||||
4011 | SmallPtrSet<SDNode *, 16> ToPromote1; | |||||
4012 | bool Op0OK = | |||||
4013 | PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); | |||||
4014 | bool Op1OK = isUInt<15>(Op32.getConstantOperandVal(1)); | |||||
4015 | if (!Op0OK && !Op1OK) | |||||
4016 | return false; | |||||
4017 | ||||||
4018 | ToPromote.insert(Op32.getNode()); | |||||
4019 | ||||||
4020 | if (Op0OK) | |||||
4021 | ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); | |||||
4022 | ||||||
4023 | return true; | |||||
4024 | } | |||||
4025 | ||||||
4026 | return false; | |||||
4027 | } | |||||
4028 | ||||||
4029 | void PPCDAGToDAGISel::PeepholePPC64ZExt() { | |||||
4030 | if (!PPCSubTarget->isPPC64()) | |||||
4031 | return; | |||||
4032 | ||||||
4033 | // When we zero-extend from i32 to i64, we use a pattern like this: | |||||
4034 | // def : Pat<(i64 (zext i32:$in)), | |||||
4035 | // (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32), | |||||
4036 | // 0, 32)>; | |||||
4037 | // There are several 32-bit shift/rotate instructions, however, that will | |||||
4038 | // clear the higher-order bits of their output, rendering the RLDICL | |||||
4039 | // unnecessary. When that happens, we remove it here, and redefine the | |||||
4040 | // relevant 32-bit operation to be a 64-bit operation. | |||||
4041 | ||||||
4042 | SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode()); | |||||
4043 | ++Position; | |||||
4044 | ||||||
4045 | bool MadeChange = false; | |||||
4046 | while (Position != CurDAG->allnodes_begin()) { | |||||
4047 | SDNode *N = &*--Position; | |||||
4048 | // Skip dead nodes and any non-machine opcodes. | |||||
4049 | if (N->use_empty() || !N->isMachineOpcode()) | |||||
4050 | continue; | |||||
4051 | ||||||
4052 | if (N->getMachineOpcode() != PPC::RLDICL) | |||||
4053 | continue; | |||||
4054 | ||||||
4055 | if (N->getConstantOperandVal(1) != 0 || | |||||
4056 | N->getConstantOperandVal(2) != 32) | |||||
4057 | continue; | |||||
4058 | ||||||
4059 | SDValue ISR = N->getOperand(0); | |||||
4060 | if (!ISR.isMachineOpcode() || | |||||
4061 | ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG) | |||||
4062 | continue; | |||||
4063 | ||||||
4064 | if (!ISR.hasOneUse()) | |||||
4065 | continue; | |||||
4066 | ||||||
4067 | if (ISR.getConstantOperandVal(2) != PPC::sub_32) | |||||
4068 | continue; | |||||
4069 | ||||||
4070 | SDValue IDef = ISR.getOperand(0); | |||||
4071 | if (!IDef.isMachineOpcode() || | |||||
4072 | IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF) | |||||
4073 | continue; | |||||
4074 | ||||||
4075 | // We now know that we're looking at a canonical i32 -> i64 zext. See if we | |||||
4076 | // can get rid of it. | |||||
4077 | ||||||
4078 | SDValue Op32 = ISR->getOperand(1); | |||||
4079 | if (!Op32.isMachineOpcode()) | |||||
4080 | continue; | |||||
4081 | ||||||
4082 | // There are some 32-bit instructions that always clear the high-order 32 | |||||
4083 | // bits, there are also some instructions (like AND) that we can look | |||||
4084 | // through. | |||||
4085 | SmallPtrSet<SDNode *, 16> ToPromote; | |||||
4086 | if (!PeepholePPC64ZExtGather(Op32, ToPromote)) | |||||
4087 | continue; | |||||
4088 | ||||||
4089 | // If the ToPromote set contains nodes that have uses outside of the set | |||||
4090 | // (except for the original INSERT_SUBREG), then abort the transformation. | |||||
4091 | bool OutsideUse = false; | |||||
4092 | for (SDNode *PN : ToPromote) { | |||||
4093 | for (SDNode *UN : PN->uses()) { | |||||
4094 | if (!ToPromote.count(UN) && UN != ISR.getNode()) { | |||||
4095 | OutsideUse = true; | |||||
4096 | break; | |||||
4097 | } | |||||
4098 | } | |||||
4099 | ||||||
4100 | if (OutsideUse) | |||||
4101 | break; | |||||
4102 | } | |||||
4103 | if (OutsideUse) | |||||
4104 | continue; | |||||
4105 | ||||||
4106 | MadeChange = true; | |||||
4107 | ||||||
4108 | // We now know that this zero extension can be removed by promoting to | |||||
4109 | // nodes in ToPromote to 64-bit operations, where for operations in the | |||||
4110 | // frontier of the set, we need to insert INSERT_SUBREGs for their | |||||
4111 | // operands. | |||||
4112 | for (SDNode *PN : ToPromote) { | |||||
4113 | unsigned NewOpcode; | |||||
4114 | switch (PN->getMachineOpcode()) { | |||||
4115 | default: | |||||
4116 | llvm_unreachable("Don't know the 64-bit variant of this instruction")::llvm::llvm_unreachable_internal("Don't know the 64-bit variant of this instruction" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn257205/lib/Target/PowerPC/PPCISelDAGToDAG.cpp" , 4116); | |||||
4117 | case PPC::RLWINM: NewOpcode = PPC::RLWINM8; break; | |||||
4118 | case PPC::RLWNM: NewOpcode = PPC::RLWNM8; break; | |||||
4119 | case PPC::SLW: NewOpcode = PPC::SLW8; break; | |||||
4120 | case PPC::SRW: NewOpcode = PPC::SRW8; break; | |||||
4121 | case PPC::LI: NewOpcode = PPC::LI8; break; | |||||
4122 | case PPC::LIS: NewOpcode = PPC::LIS8; break; | |||||
4123 | case PPC::LHBRX: NewOpcode = PPC::LHBRX8; break; | |||||
4124 | case PPC::LWBRX: NewOpcode = PPC::LWBRX8; break; | |||||
4125 | case PPC::CNTLZW: NewOpcode = PPC::CNTLZW8; break; | |||||
4126 | case PPC::RLWIMI: NewOpcode = PPC::RLWIMI8; break; | |||||
4127 | case PPC::OR: NewOpcode = PPC::OR8; break; | |||||
4128 | case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break; | |||||
4129 | case PPC::ORI: NewOpcode = PPC::ORI8; break; | |||||
4130 | case PPC::ORIS: NewOpcode = PPC::ORIS8; break; | |||||
4131 | case PPC::AND: NewOpcode = PPC::AND8; break; | |||||
4132 | case PPC::ANDIo: NewOpcode = PPC::ANDIo8; break; | |||||
4133 | case PPC::ANDISo: NewOpcode = PPC::ANDISo8; break; | |||||
4134 | } | |||||
4135 | ||||||
4136 | // Note: During the replacement process, the nodes will be in an | |||||
4137 | // inconsistent state (some instructions will have operands with values | |||||
4138 | // of the wrong type). Once done, however, everything should be right | |||||
4139 | // again. | |||||
4140 | ||||||
4141 | SmallVector<SDValue, 4> Ops; | |||||
4142 | for (const SDValue &V : PN->ops()) { | |||||
4143 | if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 && | |||||
4144 | !isa<ConstantSDNode>(V)) { | |||||
4145 | SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) }; | |||||
4146 | SDNode *ReplOp = | |||||
4147 | CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V), | |||||
4148 | ISR.getNode()->getVTList(), ReplOpOps); | |||||
4149 | Ops.push_back(SDValue(ReplOp, 0)); | |||||
4150 | } else { | |||||
4151 | Ops.push_back(V); | |||||
4152 | } | |||||
4153 | } | |||||
4154 | ||||||
4155 | // Because all to-be-promoted nodes only have users that are other | |||||
4156 | // promoted nodes (or the original INSERT_SUBREG), we can safely replace | |||||
4157 | // the i32 result value type with i64. | |||||
4158 | ||||||
4159 | SmallVector<EVT, 2> NewVTs; | |||||
4160 | SDVTList VTs = PN->getVTList(); | |||||
4161 | for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i) | |||||
4162 | if (VTs.VTs[i] == MVT::i32) | |||||
4163 | NewVTs.push_back(MVT::i64); | |||||
4164 | else | |||||
4165 | NewVTs.push_back(VTs.VTs[i]); | |||||
4166 | ||||||
4167 | DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "PPC64 ZExt Peephole morphing:\nOld: " ; } } while (0); | |||||
4168 | DEBUG(PN->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { PN->dump(CurDAG); } } while (0); | |||||
4169 | ||||||
4170 | CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops); | |||||
4171 | ||||||
4172 | DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\nNew: "; } } while (0); | |||||
4173 | DEBUG(PN->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { PN->dump(CurDAG); } } while (0); | |||||
4174 | DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\n"; } } while (0); | |||||
4175 | } | |||||
4176 | ||||||
4177 | // Now we replace the original zero extend and its associated INSERT_SUBREG | |||||
4178 | // with the value feeding the INSERT_SUBREG (which has now been promoted to | |||||
4179 | // return an i64). | |||||
4180 | ||||||
4181 | DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "PPC64 ZExt Peephole replacing:\nOld: " ; } } while (0); | |||||
4182 | DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { N->dump(CurDAG); } } while (0); | |||||
4183 | DEBUG(dbgs() << "\nNew: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\nNew: "; } } while (0); | |||||
4184 | DEBUG(Op32.getNode()->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { Op32.getNode()->dump(CurDAG); } } while (0); | |||||
4185 | DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\n"; } } while (0); | |||||
4186 | ||||||
4187 | ReplaceUses(N, Op32.getNode()); | |||||
4188 | } | |||||
4189 | ||||||
4190 | if (MadeChange) | |||||
4191 | CurDAG->RemoveDeadNodes(); | |||||
4192 | } | |||||
4193 | ||||||
4194 | void PPCDAGToDAGISel::PeepholePPC64() { | |||||
4195 | // These optimizations are currently supported only for 64-bit SVR4. | |||||
4196 | if (PPCSubTarget->isDarwin() || !PPCSubTarget->isPPC64()) | |||||
4197 | return; | |||||
4198 | ||||||
4199 | SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode()); | |||||
4200 | ++Position; | |||||
4201 | ||||||
4202 | while (Position != CurDAG->allnodes_begin()) { | |||||
4203 | SDNode *N = &*--Position; | |||||
4204 | // Skip dead nodes and any non-machine opcodes. | |||||
4205 | if (N->use_empty() || !N->isMachineOpcode()) | |||||
4206 | continue; | |||||
4207 | ||||||
4208 | unsigned FirstOp; | |||||
4209 | unsigned StorageOpcode = N->getMachineOpcode(); | |||||
4210 | ||||||
4211 | switch (StorageOpcode) { | |||||
4212 | default: continue; | |||||
4213 | ||||||
4214 | case PPC::LBZ: | |||||
4215 | case PPC::LBZ8: | |||||
4216 | case PPC::LD: | |||||
4217 | case PPC::LFD: | |||||
4218 | case PPC::LFS: | |||||
4219 | case PPC::LHA: | |||||
4220 | case PPC::LHA8: | |||||
4221 | case PPC::LHZ: | |||||
4222 | case PPC::LHZ8: | |||||
4223 | case PPC::LWA: | |||||
4224 | case PPC::LWZ: | |||||
4225 | case PPC::LWZ8: | |||||
4226 | FirstOp = 0; | |||||
4227 | break; | |||||
4228 | ||||||
4229 | case PPC::STB: | |||||
4230 | case PPC::STB8: | |||||
4231 | case PPC::STD: | |||||
4232 | case PPC::STFD: | |||||
4233 | case PPC::STFS: | |||||
4234 | case PPC::STH: | |||||
4235 | case PPC::STH8: | |||||
4236 | case PPC::STW: | |||||
4237 | case PPC::STW8: | |||||
4238 | FirstOp = 1; | |||||
4239 | break; | |||||
4240 | } | |||||
4241 | ||||||
4242 | // If this is a load or store with a zero offset, or within the alignment, | |||||
4243 | // we may be able to fold an add-immediate into the memory operation. | |||||
4244 | // The check against alignment is below, as it can't occur until we check | |||||
4245 | // the arguments to N | |||||
4246 | if (!isa<ConstantSDNode>(N->getOperand(FirstOp))) | |||||
4247 | continue; | |||||
4248 | ||||||
4249 | SDValue Base = N->getOperand(FirstOp + 1); | |||||
4250 | if (!Base.isMachineOpcode()) | |||||
4251 | continue; | |||||
4252 | ||||||
4253 | // On targets with fusion, we don't want this to fire and remove a fusion | |||||
4254 | // opportunity, unless a) it results in another fusion opportunity or | |||||
4255 | // b) optimizing for size. | |||||
4256 | if (PPCSubTarget->hasFusion() && | |||||
4257 | (!MF->getFunction()->optForSize() && !Base.hasOneUse())) | |||||
4258 | continue; | |||||
4259 | ||||||
4260 | unsigned Flags = 0; | |||||
4261 | bool ReplaceFlags = true; | |||||
4262 | ||||||
4263 | // When the feeding operation is an add-immediate of some sort, | |||||
4264 | // determine whether we need to add relocation information to the | |||||
4265 | // target flags on the immediate operand when we fold it into the | |||||
4266 | // load instruction. | |||||
4267 | // | |||||
4268 | // For something like ADDItocL, the relocation information is | |||||
4269 | // inferred from the opcode; when we process it in the AsmPrinter, | |||||
4270 | // we add the necessary relocation there. A load, though, can receive | |||||
4271 | // relocation from various flavors of ADDIxxx, so we need to carry | |||||
4272 | // the relocation information in the target flags. | |||||
4273 | switch (Base.getMachineOpcode()) { | |||||
4274 | default: continue; | |||||
4275 | ||||||
4276 | case PPC::ADDI8: | |||||
4277 | case PPC::ADDI: | |||||
4278 | // In some cases (such as TLS) the relocation information | |||||
4279 | // is already in place on the operand, so copying the operand | |||||
4280 | // is sufficient. | |||||
4281 | ReplaceFlags = false; | |||||
4282 | // For these cases, the immediate may not be divisible by 4, in | |||||
4283 | // which case the fold is illegal for DS-form instructions. (The | |||||
4284 | // other cases provide aligned addresses and are always safe.) | |||||
4285 | if ((StorageOpcode == PPC::LWA || | |||||
4286 | StorageOpcode == PPC::LD || | |||||
4287 | StorageOpcode == PPC::STD) && | |||||
4288 | (!isa<ConstantSDNode>(Base.getOperand(1)) || | |||||
4289 | Base.getConstantOperandVal(1) % 4 != 0)) | |||||
4290 | continue; | |||||
4291 | break; | |||||
4292 | case PPC::ADDIdtprelL: | |||||
4293 | Flags = PPCII::MO_DTPREL_LO; | |||||
4294 | break; | |||||
4295 | case PPC::ADDItlsldL: | |||||
4296 | Flags = PPCII::MO_TLSLD_LO; | |||||
4297 | break; | |||||
4298 | case PPC::ADDItocL: | |||||
4299 | Flags = PPCII::MO_TOC_LO; | |||||
4300 | break; | |||||
4301 | } | |||||
4302 | ||||||
4303 | SDValue ImmOpnd = Base.getOperand(1); | |||||
4304 | int MaxDisplacement = 0; | |||||
4305 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) { | |||||
4306 | const GlobalValue *GV = GA->getGlobal(); | |||||
4307 | MaxDisplacement = GV->getAlignment() - 1; | |||||
4308 | } | |||||
4309 | ||||||
4310 | int Offset = N->getConstantOperandVal(FirstOp); | |||||
4311 | if (Offset < 0 || Offset > MaxDisplacement) | |||||
4312 | continue; | |||||
4313 | ||||||
4314 | // We found an opportunity. Reverse the operands from the add | |||||
4315 | // immediate and substitute them into the load or store. If | |||||
4316 | // needed, update the target flags for the immediate operand to | |||||
4317 | // reflect the necessary relocation information. | |||||
4318 | DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Folding add-immediate into mem-op:\nBase: " ; } } while (0); | |||||
4319 | DEBUG(Base->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { Base->dump(CurDAG); } } while (0); | |||||
4320 | DEBUG(dbgs() << "\nN: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\nN: "; } } while (0); | |||||
4321 | DEBUG(N->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { N->dump(CurDAG); } } while (0); | |||||
4322 | DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "\n"; } } while (0); | |||||
4323 | ||||||
4324 | // If the relocation information isn't already present on the | |||||
4325 | // immediate operand, add it now. | |||||
4326 | if (ReplaceFlags) { | |||||
4327 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) { | |||||
4328 | SDLoc dl(GA); | |||||
4329 | const GlobalValue *GV = GA->getGlobal(); | |||||
4330 | // We can't perform this optimization for data whose alignment | |||||
4331 | // is insufficient for the instruction encoding. | |||||
4332 | if (GV->getAlignment() < 4 && | |||||
4333 | (StorageOpcode == PPC::LD || StorageOpcode == PPC::STD || | |||||
4334 | StorageOpcode == PPC::LWA || (Offset % 4) != 0)) { | |||||
4335 | DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("ppc-codegen")) { dbgs() << "Rejected this candidate for alignment.\n\n" ; } } while (0); | |||||
4336 | continue; | |||||
4337 | } | |||||
4338 | ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags); | |||||
4339 | } else if (ConstantPoolSDNode *CP = | |||||
4340 | dyn_cast<ConstantPoolSDNode>(ImmOpnd)) { | |||||
4341 | const Constant *C = CP->getConstVal(); | |||||
4342 | ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, | |||||
4343 | CP->getAlignment(), | |||||
4344 | Offset, Flags); | |||||
4345 | } | |||||
4346 | } | |||||
4347 | ||||||
4348 | if (FirstOp == 1) // Store | |||||
4349 | (void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd, | |||||
4350 | Base.getOperand(0), N->getOperand(3)); | |||||
4351 | else // Load | |||||
4352 | (void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0), | |||||
4353 | N->getOperand(2)); | |||||
4354 | ||||||
4355 | // The add-immediate may now be dead, in which case remove it. | |||||
4356 | if (Base.getNode()->use_empty()) | |||||
4357 | CurDAG->RemoveDeadNode(Base.getNode()); | |||||
4358 | } | |||||
4359 | } | |||||
4360 | ||||||
4361 | ||||||
4362 | /// createPPCISelDag - This pass converts a legalized DAG into a | |||||
4363 | /// PowerPC-specific DAG, ready for instruction scheduling. | |||||
4364 | /// | |||||
4365 | FunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM) { | |||||
4366 | return new PPCDAGToDAGISel(TM); | |||||
4367 | } | |||||
4368 | ||||||
4369 | static void initializePassOnce(PassRegistry &Registry) { | |||||
4370 | const char *Name = "PowerPC DAG->DAG Pattern Instruction Selection"; | |||||
4371 | PassInfo *PI = new PassInfo(Name, "ppc-codegen", &SelectionDAGISel::ID, | |||||
4372 | nullptr, false, false); | |||||
4373 | Registry.registerPass(*PI, true); | |||||
4374 | } | |||||
4375 | ||||||
4376 | void llvm::initializePPCDAGToDAGISelPass(PassRegistry &Registry) { | |||||
4377 | CALL_ONCE_INITIALIZATION(initializePassOnce)static volatile sys::cas_flag initialized = 0; sys::cas_flag old_val = sys::CompareAndSwap(&initialized, 1, 0); if (old_val == 0) { initializePassOnce(Registry); sys::MemoryFence(); ; ; initialized = 2; ; } else { sys::cas_flag tmp = initialized; sys::MemoryFence (); while (tmp != 2) { tmp = initialized; sys::MemoryFence(); } } ;; | |||||
4378 | } | |||||
4379 |