Line data Source code
1 : //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements the SelectionDAG::Legalize method.
11 : //
12 : //===----------------------------------------------------------------------===//
13 :
14 : #include "llvm/ADT/APFloat.h"
15 : #include "llvm/ADT/APInt.h"
16 : #include "llvm/ADT/ArrayRef.h"
17 : #include "llvm/ADT/SetVector.h"
18 : #include "llvm/ADT/SmallPtrSet.h"
19 : #include "llvm/ADT/SmallSet.h"
20 : #include "llvm/ADT/SmallVector.h"
21 : #include "llvm/CodeGen/ISDOpcodes.h"
22 : #include "llvm/CodeGen/MachineFunction.h"
23 : #include "llvm/CodeGen/MachineJumpTableInfo.h"
24 : #include "llvm/CodeGen/MachineMemOperand.h"
25 : #include "llvm/CodeGen/RuntimeLibcalls.h"
26 : #include "llvm/CodeGen/SelectionDAG.h"
27 : #include "llvm/CodeGen/SelectionDAGNodes.h"
28 : #include "llvm/CodeGen/TargetFrameLowering.h"
29 : #include "llvm/CodeGen/TargetLowering.h"
30 : #include "llvm/CodeGen/TargetSubtargetInfo.h"
31 : #include "llvm/CodeGen/ValueTypes.h"
32 : #include "llvm/IR/CallingConv.h"
33 : #include "llvm/IR/Constants.h"
34 : #include "llvm/IR/DataLayout.h"
35 : #include "llvm/IR/DerivedTypes.h"
36 : #include "llvm/IR/Function.h"
37 : #include "llvm/IR/Metadata.h"
38 : #include "llvm/IR/Type.h"
39 : #include "llvm/Support/Casting.h"
40 : #include "llvm/Support/Compiler.h"
41 : #include "llvm/Support/Debug.h"
42 : #include "llvm/Support/ErrorHandling.h"
43 : #include "llvm/Support/MachineValueType.h"
44 : #include "llvm/Support/MathExtras.h"
45 : #include "llvm/Support/raw_ostream.h"
46 : #include "llvm/Target/TargetMachine.h"
47 : #include "llvm/Target/TargetOptions.h"
48 : #include <algorithm>
49 : #include <cassert>
50 : #include <cstdint>
51 : #include <tuple>
52 : #include <utility>
53 :
54 : using namespace llvm;
55 :
56 : #define DEBUG_TYPE "legalizedag"
57 :
58 : namespace {
59 :
60 : /// Keeps track of state when getting the sign of a floating-point value as an
61 : /// integer.
62 242 : struct FloatSignAsInt {
63 : EVT FloatVT;
64 : SDValue Chain;
65 : SDValue FloatPtr;
66 : SDValue IntPtr;
67 : MachinePointerInfo IntPointerInfo;
68 : MachinePointerInfo FloatPointerInfo;
69 : SDValue IntValue;
70 : APInt SignMask;
71 : uint8_t SignBit;
72 : };
73 :
74 : //===----------------------------------------------------------------------===//
75 : /// This takes an arbitrary SelectionDAG as input and
76 : /// hacks on it until the target machine can handle it. This involves
77 : /// eliminating value sizes the machine cannot handle (promoting small sizes to
78 : /// large sizes or splitting up large values into small values) as well as
79 : /// eliminating operations the machine cannot handle.
80 : ///
81 : /// This code also does a small amount of optimization and recognition of idioms
82 : /// as part of its processing. For example, if a target does not support a
83 : /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84 : /// will attempt merge setcc and brc instructions into brcc's.
85 : class SelectionDAGLegalize {
86 : const TargetMachine &TM;
87 : const TargetLowering &TLI;
88 : SelectionDAG &DAG;
89 :
90 : /// The set of nodes which have already been legalized. We hold a
91 : /// reference to it in order to update as necessary on node deletion.
92 : SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93 :
94 : /// A set of all the nodes updated during legalization.
95 : SmallSetVector<SDNode *, 16> *UpdatedNodes;
96 :
97 0 : EVT getSetCCResultType(EVT VT) const {
98 0 : return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99 : }
100 :
101 : // Libcall insertion helpers.
102 :
103 : public:
104 : SelectionDAGLegalize(SelectionDAG &DAG,
105 : SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106 : SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107 80621756 : : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108 80621756 : LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109 :
110 : /// Legalizes the given operation.
111 : void LegalizeOp(SDNode *Node);
112 :
113 : private:
114 : SDValue OptimizeFloatStore(StoreSDNode *ST);
115 :
116 : void LegalizeLoadOps(SDNode *Node);
117 : void LegalizeStoreOps(SDNode *Node);
118 :
119 : /// Some targets cannot handle a variable
120 : /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121 : /// is necessary to spill the vector being inserted into to memory, perform
122 : /// the insert there, and then read the result back.
123 : SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124 : const SDLoc &dl);
125 : SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126 : const SDLoc &dl);
127 :
128 : /// Return a vector shuffle operation which
129 : /// performs the same shuffe in terms of order or result bytes, but on a type
130 : /// whose vector element type is narrower than the original shuffle type.
131 : /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132 : SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133 : SDValue N1, SDValue N2,
134 : ArrayRef<int> Mask) const;
135 :
136 : bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137 : bool &NeedInvert, const SDLoc &dl);
138 :
139 : SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 : SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141 : unsigned NumOps, bool isSigned, const SDLoc &dl);
142 :
143 : std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144 : SDNode *Node, bool isSigned);
145 : SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 : RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 : RTLIB::Libcall Call_F128,
148 : RTLIB::Libcall Call_PPCF128);
149 : SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 : RTLIB::Libcall Call_I8,
151 : RTLIB::Libcall Call_I16,
152 : RTLIB::Libcall Call_I32,
153 : RTLIB::Libcall Call_I64,
154 : RTLIB::Libcall Call_I128);
155 : void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 : void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 :
158 : SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159 : const SDLoc &dl);
160 : SDValue ExpandBUILD_VECTOR(SDNode *Node);
161 : SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162 : void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163 : SmallVectorImpl<SDValue> &Results);
164 : void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165 : SDValue Value) const;
166 : SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167 : SDValue NewIntValue) const;
168 : SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169 : SDValue ExpandFABS(SDNode *Node) const;
170 : SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
171 : const SDLoc &dl);
172 : SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173 : const SDLoc &dl);
174 : SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175 : const SDLoc &dl);
176 :
177 : SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178 : SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179 : SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
180 :
181 : SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
182 : SDValue ExpandInsertToVectorThroughStack(SDValue Op);
183 : SDValue ExpandVectorBuildThroughStack(SDNode* Node);
184 :
185 : SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
186 : SDValue ExpandConstant(ConstantSDNode *CP);
187 :
188 : // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
189 : bool ExpandNode(SDNode *Node);
190 : void ConvertNodeToLibcall(SDNode *Node);
191 : void PromoteNode(SDNode *Node);
192 :
193 : public:
194 : // Node replacement helpers
195 :
196 0 : void ReplacedNode(SDNode *N) {
197 0 : LegalizedNodes.erase(N);
198 0 : if (UpdatedNodes)
199 0 : UpdatedNodes->insert(N);
200 0 : }
201 :
202 136 : void ReplaceNode(SDNode *Old, SDNode *New) {
203 : LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
204 : dbgs() << " with: "; New->dump(&DAG));
205 :
206 : assert(Old->getNumValues() == New->getNumValues() &&
207 : "Replacing one node with another that produces a different number "
208 : "of values!");
209 136 : DAG.ReplaceAllUsesWith(Old, New);
210 136 : if (UpdatedNodes)
211 1 : UpdatedNodes->insert(New);
212 136 : ReplacedNode(Old);
213 136 : }
214 :
215 1151087 : void ReplaceNode(SDValue Old, SDValue New) {
216 : LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
217 : dbgs() << " with: "; New->dump(&DAG));
218 :
219 1151087 : DAG.ReplaceAllUsesWith(Old, New);
220 1151087 : if (UpdatedNodes)
221 4169 : UpdatedNodes->insert(New.getNode());
222 1151087 : ReplacedNode(Old.getNode());
223 1151087 : }
224 :
225 543953 : void ReplaceNode(SDNode *Old, const SDValue *New) {
226 : LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
227 :
228 543953 : DAG.ReplaceAllUsesWith(Old, New);
229 1558668 : for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
230 : LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
231 : New[i]->dump(&DAG));
232 1014715 : if (UpdatedNodes)
233 4385 : UpdatedNodes->insert(New[i].getNode());
234 : }
235 543953 : ReplacedNode(Old);
236 543953 : }
237 : };
238 :
239 : } // end anonymous namespace
240 :
241 : /// Return a vector shuffle operation which
242 : /// performs the same shuffe in terms of order or result bytes, but on a type
243 : /// whose vector element type is narrower than the original shuffle type.
244 : /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
245 0 : SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
246 : EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
247 : ArrayRef<int> Mask) const {
248 : unsigned NumMaskElts = VT.getVectorNumElements();
249 : unsigned NumDestElts = NVT.getVectorNumElements();
250 0 : unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
251 :
252 : assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
253 :
254 0 : if (NumEltsGrowth == 1)
255 0 : return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
256 :
257 : SmallVector<int, 8> NewMask;
258 0 : for (unsigned i = 0; i != NumMaskElts; ++i) {
259 0 : int Idx = Mask[i];
260 0 : for (unsigned j = 0; j != NumEltsGrowth; ++j) {
261 0 : if (Idx < 0)
262 0 : NewMask.push_back(-1);
263 : else
264 0 : NewMask.push_back(Idx * NumEltsGrowth + j);
265 : }
266 : }
267 : assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
268 : assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
269 0 : return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
270 : }
271 :
272 : /// Expands the ConstantFP node to an integer constant or
273 : /// a load from the constant pool.
274 : SDValue
275 7285 : SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
276 : bool Extend = false;
277 : SDLoc dl(CFP);
278 :
279 : // If a FP immediate is precise when represented as a float and if the
280 : // target can do an extending load from float to double, we put it into
281 : // the constant pool as a float, even if it's is statically typed as a
282 : // double. This shrinks FP constants and canonicalizes them for targets where
283 : // an FP extending load is the same cost as a normal load (such as on the x87
284 : // fp stack or PPC FP unit).
285 7285 : EVT VT = CFP->getValueType(0);
286 7285 : ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
287 7285 : if (!UseCP) {
288 : assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
289 0 : return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
290 0 : (VT == MVT::f64) ? MVT::i64 : MVT::i32);
291 : }
292 :
293 : APFloat APF = CFP->getValueAPF();
294 7285 : EVT OrigVT = VT;
295 7285 : EVT SVT = VT;
296 :
297 : // We don't want to shrink SNaNs. Converting the SNaN back to its real type
298 : // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
299 7285 : if (!APF.isSignaling()) {
300 11507 : while (SVT != MVT::f32 && SVT != MVT::f16) {
301 4228 : SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
302 7736 : if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
303 : // Only do this if the target has a native EXTLOAD instruction from
304 : // smaller type.
305 7410 : TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
306 3182 : TLI.ShouldShrinkFPConstant(OrigVT)) {
307 2071 : Type *SType = SVT.getTypeForEVT(*DAG.getContext());
308 2071 : LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
309 2071 : VT = SVT;
310 : Extend = true;
311 : }
312 : }
313 : }
314 :
315 : SDValue CPIdx =
316 14570 : DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
317 7285 : unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
318 7285 : if (Extend) {
319 1230 : SDValue Result = DAG.getExtLoad(
320 1230 : ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
321 : MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
322 1230 : Alignment);
323 1230 : return Result;
324 : }
325 6055 : SDValue Result = DAG.getLoad(
326 6055 : OrigVT, dl, DAG.getEntryNode(), CPIdx,
327 6055 : MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
328 6055 : return Result;
329 : }
330 :
331 : /// Expands the Constant node to a load from the constant pool.
332 0 : SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
333 : SDLoc dl(CP);
334 0 : EVT VT = CP->getValueType(0);
335 0 : SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
336 0 : TLI.getPointerTy(DAG.getDataLayout()));
337 0 : unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
338 0 : SDValue Result = DAG.getLoad(
339 0 : VT, dl, DAG.getEntryNode(), CPIdx,
340 0 : MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
341 0 : return Result;
342 : }
343 :
344 : /// Some target cannot handle a variable insertion index for the
345 : /// INSERT_VECTOR_ELT instruction. In this case, it
346 : /// is necessary to spill the vector being inserted into to memory, perform
347 : /// the insert there, and then read the result back.
348 0 : SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
349 : SDValue Val,
350 : SDValue Idx,
351 : const SDLoc &dl) {
352 0 : SDValue Tmp1 = Vec;
353 0 : SDValue Tmp2 = Val;
354 0 : SDValue Tmp3 = Idx;
355 :
356 : // If the target doesn't support this, we have to spill the input vector
357 : // to a temporary stack slot, update the element, then reload it. This is
358 : // badness. We could also load the value into a vector register (either
359 : // with a "move to register" or "extload into register" instruction, then
360 : // permute it into place, if the idx is a constant and if the idx is
361 : // supported by the target.
362 0 : EVT VT = Tmp1.getValueType();
363 0 : EVT EltVT = VT.getVectorElementType();
364 0 : SDValue StackPtr = DAG.CreateStackTemporary(VT);
365 :
366 0 : int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
367 :
368 : // Store the vector.
369 0 : SDValue Ch = DAG.getStore(
370 0 : DAG.getEntryNode(), dl, Tmp1, StackPtr,
371 0 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
372 :
373 0 : SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
374 :
375 : // Store the scalar value.
376 0 : Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
377 : // Load the updated vector.
378 0 : return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
379 0 : DAG.getMachineFunction(), SPFI));
380 : }
381 :
382 1411 : SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
383 : SDValue Idx,
384 : const SDLoc &dl) {
385 : if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
386 : // SCALAR_TO_VECTOR requires that the type of the value being inserted
387 : // match the element type of the vector being created, except for
388 : // integers in which case the inserted value can be over width.
389 1383 : EVT EltVT = Vec.getValueType().getVectorElementType();
390 1383 : if (Val.getValueType() == EltVT ||
391 1 : (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
392 1383 : SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
393 1383 : Vec.getValueType(), Val);
394 :
395 2766 : unsigned NumElts = Vec.getValueType().getVectorNumElements();
396 : // We generate a shuffle of InVec and ScVec, so the shuffle mask
397 : // should be 0,1,2,3,4,5... with the appropriate element replaced with
398 : // elt 0 of the RHS.
399 : SmallVector<int, 8> ShufOps;
400 5465 : for (unsigned i = 0; i != NumElts; ++i)
401 8164 : ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
402 :
403 2766 : return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
404 : }
405 : }
406 28 : return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
407 : }
408 :
409 0 : SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
410 : LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
411 : // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
412 : // FIXME: We shouldn't do this for TargetConstantFP's.
413 : // FIXME: move this to the DAG Combiner! Note that we can't regress due
414 : // to phase ordering between legalized code and the dag combiner. This
415 : // probably means that we need to integrate dag combiner and legalizer
416 : // together.
417 : // We generally can't do this one for long doubles.
418 0 : SDValue Chain = ST->getChain();
419 0 : SDValue Ptr = ST->getBasePtr();
420 0 : unsigned Alignment = ST->getAlignment();
421 0 : MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
422 : AAMDNodes AAInfo = ST->getAAInfo();
423 : SDLoc dl(ST);
424 : if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
425 0 : if (CFP->getValueType(0) == MVT::f32 &&
426 0 : TLI.isTypeLegal(MVT::i32)) {
427 0 : SDValue Con = DAG.getConstant(CFP->getValueAPF().
428 0 : bitcastToAPInt().zextOrTrunc(32),
429 0 : SDLoc(CFP), MVT::i32);
430 0 : return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
431 0 : MMOFlags, AAInfo);
432 : }
433 :
434 0 : if (CFP->getValueType(0) == MVT::f64) {
435 : // If this target supports 64-bit registers, do a single 64-bit store.
436 0 : if (TLI.isTypeLegal(MVT::i64)) {
437 0 : SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
438 0 : zextOrTrunc(64), SDLoc(CFP), MVT::i64);
439 0 : return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
440 0 : Alignment, MMOFlags, AAInfo);
441 : }
442 :
443 0 : if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
444 : // Otherwise, if the target supports 32-bit registers, use 2 32-bit
445 : // stores. If the target supports neither 32- nor 64-bits, this
446 : // xform is certainly not worth it.
447 0 : const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
448 0 : SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
449 0 : SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
450 0 : if (DAG.getDataLayout().isBigEndian())
451 : std::swap(Lo, Hi);
452 :
453 0 : Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
454 0 : MMOFlags, AAInfo);
455 0 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
456 0 : DAG.getConstant(4, dl, Ptr.getValueType()));
457 0 : Hi = DAG.getStore(Chain, dl, Hi, Ptr,
458 0 : ST->getPointerInfo().getWithOffset(4),
459 0 : MinAlign(Alignment, 4U), MMOFlags, AAInfo);
460 :
461 0 : return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
462 : }
463 : }
464 : }
465 0 : return SDValue(nullptr, 0);
466 : }
467 :
468 5859005 : void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
469 : StoreSDNode *ST = cast<StoreSDNode>(Node);
470 5859005 : SDValue Chain = ST->getChain();
471 5859005 : SDValue Ptr = ST->getBasePtr();
472 : SDLoc dl(Node);
473 :
474 5859005 : unsigned Alignment = ST->getAlignment();
475 5859005 : MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
476 : AAMDNodes AAInfo = ST->getAAInfo();
477 :
478 5859005 : if (!ST->isTruncatingStore()) {
479 : LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
480 5822861 : if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
481 1 : ReplaceNode(ST, OptStore);
482 1 : return;
483 : }
484 :
485 5822860 : SDValue Value = ST->getValue();
486 : MVT VT = Value.getSimpleValueType();
487 5822860 : switch (TLI.getOperationAction(ISD::STORE, VT)) {
488 0 : default: llvm_unreachable("This action is not supported yet!");
489 5687807 : case TargetLowering::Legal: {
490 : // If this is an unaligned store and the target doesn't support it,
491 : // expand it.
492 5687807 : EVT MemVT = ST->getMemoryVT();
493 : unsigned AS = ST->getAddressSpace();
494 5687807 : unsigned Align = ST->getAlignment();
495 5687807 : const DataLayout &DL = DAG.getDataLayout();
496 5687807 : if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
497 : LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
498 217 : SDValue Result = TLI.expandUnalignedStore(ST, DAG);
499 217 : ReplaceNode(SDValue(ST, 0), Result);
500 : } else
501 : LLVM_DEBUG(dbgs() << "Legal store\n");
502 : break;
503 : }
504 124280 : case TargetLowering::Custom: {
505 : LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
506 248560 : SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
507 124279 : if (Res && Res != SDValue(Node, 0))
508 15376 : ReplaceNode(SDValue(Node, 0), Res);
509 : return;
510 : }
511 10773 : case TargetLowering::Promote: {
512 10773 : MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
513 : assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
514 : "Can only promote stores to same size type");
515 21546 : Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
516 : SDValue Result =
517 21546 : DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
518 21546 : Alignment, MMOFlags, AAInfo);
519 10773 : ReplaceNode(SDValue(Node, 0), Result);
520 : break;
521 : }
522 : }
523 5698580 : return;
524 : }
525 :
526 : LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
527 36144 : SDValue Value = ST->getValue();
528 36144 : EVT StVT = ST->getMemoryVT();
529 36144 : unsigned StWidth = StVT.getSizeInBits();
530 36144 : auto &DL = DAG.getDataLayout();
531 :
532 36144 : if (StWidth != StVT.getStoreSizeInBits()) {
533 : // Promote to a byte-sized store with upper bits zero if not
534 : // storing an integral number of bytes. For example, promote
535 : // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
536 5399 : EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
537 5399 : StVT.getStoreSizeInBits());
538 5399 : Value = DAG.getZeroExtendInReg(Value, dl, StVT);
539 : SDValue Result =
540 10798 : DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
541 10798 : Alignment, MMOFlags, AAInfo);
542 5399 : ReplaceNode(SDValue(Node, 0), Result);
543 30745 : } else if (StWidth & (StWidth - 1)) {
544 : // If not storing a power-of-2 number of bits, expand as two stores.
545 : assert(!StVT.isVector() && "Unsupported truncstore!");
546 : unsigned RoundWidth = 1 << Log2_32(StWidth);
547 : assert(RoundWidth < StWidth);
548 744 : unsigned ExtraWidth = StWidth - RoundWidth;
549 : assert(ExtraWidth < RoundWidth);
550 : assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
551 : "Store size not an integral number of bytes!");
552 744 : EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
553 744 : EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
554 744 : SDValue Lo, Hi;
555 : unsigned IncrementSize;
556 :
557 744 : if (DL.isLittleEndian()) {
558 : // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
559 : // Store the bottom RoundWidth bits.
560 2196 : Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
561 1464 : RoundVT, Alignment, MMOFlags, AAInfo);
562 :
563 : // Store the remaining ExtraWidth bits.
564 732 : IncrementSize = RoundWidth / 8;
565 732 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
566 : DAG.getConstant(IncrementSize, dl,
567 732 : Ptr.getValueType()));
568 732 : Hi = DAG.getNode(
569 : ISD::SRL, dl, Value.getValueType(), Value,
570 : DAG.getConstant(RoundWidth, dl,
571 1464 : TLI.getShiftAmountTy(Value.getValueType(), DL)));
572 1464 : Hi = DAG.getTruncStore(
573 : Chain, dl, Hi, Ptr,
574 732 : ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
575 1464 : MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
576 : } else {
577 : // Big endian - avoid unaligned stores.
578 : // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
579 : // Store the top RoundWidth bits.
580 12 : Hi = DAG.getNode(
581 : ISD::SRL, dl, Value.getValueType(), Value,
582 : DAG.getConstant(ExtraWidth, dl,
583 24 : TLI.getShiftAmountTy(Value.getValueType(), DL)));
584 36 : Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
585 24 : RoundVT, Alignment, MMOFlags, AAInfo);
586 :
587 : // Store the remaining ExtraWidth bits.
588 12 : IncrementSize = RoundWidth / 8;
589 12 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
590 : DAG.getConstant(IncrementSize, dl,
591 12 : Ptr.getValueType()));
592 24 : Lo = DAG.getTruncStore(
593 : Chain, dl, Value, Ptr,
594 12 : ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
595 24 : MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
596 : }
597 :
598 : // The order of the stores doesn't matter.
599 1488 : SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
600 744 : ReplaceNode(SDValue(Node, 0), Result);
601 : } else {
602 120004 : switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
603 0 : default: llvm_unreachable("This action is not supported yet!");
604 26655 : case TargetLowering::Legal: {
605 26655 : EVT MemVT = ST->getMemoryVT();
606 : unsigned AS = ST->getAddressSpace();
607 26655 : unsigned Align = ST->getAlignment();
608 : // If this is an unaligned store and the target doesn't support it,
609 : // expand it.
610 26655 : if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
611 440 : SDValue Result = TLI.expandUnalignedStore(ST, DAG);
612 440 : ReplaceNode(SDValue(ST, 0), Result);
613 : }
614 : break;
615 : }
616 1718 : case TargetLowering::Custom: {
617 3436 : SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
618 1718 : if (Res && Res != SDValue(Node, 0))
619 1552 : ReplaceNode(SDValue(Node, 0), Res);
620 : return;
621 : }
622 1628 : case TargetLowering::Expand:
623 : assert(!StVT.isVector() &&
624 : "Vector Stores are handled in LegalizeVectorOps");
625 :
626 1628 : SDValue Result;
627 :
628 : // TRUNCSTORE:i16 i32 -> STORE i16
629 : if (TLI.isTypeLegal(StVT)) {
630 3224 : Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
631 1612 : Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
632 3224 : Alignment, MMOFlags, AAInfo);
633 : } else {
634 : // The in-memory type isn't legal. Truncate to the type it would promote
635 : // to, and then do a truncstore.
636 16 : Value = DAG.getNode(ISD::TRUNCATE, dl,
637 16 : TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
638 16 : Value);
639 16 : Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
640 32 : StVT, Alignment, MMOFlags, AAInfo);
641 : }
642 :
643 1628 : ReplaceNode(SDValue(Node, 0), Result);
644 1628 : break;
645 : }
646 : }
647 : }
648 :
649 5211989 : void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
650 : LoadSDNode *LD = cast<LoadSDNode>(Node);
651 5211989 : SDValue Chain = LD->getChain(); // The chain.
652 5211989 : SDValue Ptr = LD->getBasePtr(); // The base pointer.
653 5211989 : SDValue Value; // The value returned by the load op.
654 : SDLoc dl(Node);
655 :
656 : ISD::LoadExtType ExtType = LD->getExtensionType();
657 5211989 : if (ExtType == ISD::NON_EXTLOAD) {
658 : LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
659 : MVT VT = Node->getSimpleValueType(0);
660 : SDValue RVal = SDValue(Node, 0);
661 : SDValue RChain = SDValue(Node, 1);
662 :
663 10223754 : switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
664 0 : default: llvm_unreachable("This action is not supported yet!");
665 4911406 : case TargetLowering::Legal: {
666 4911406 : EVT MemVT = LD->getMemoryVT();
667 : unsigned AS = LD->getAddressSpace();
668 4911406 : unsigned Align = LD->getAlignment();
669 4911406 : const DataLayout &DL = DAG.getDataLayout();
670 : // If this is an unaligned load and the target doesn't support it,
671 : // expand it.
672 4911406 : if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
673 205 : std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
674 : }
675 : break;
676 : }
677 126751 : case TargetLowering::Custom:
678 126751 : if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
679 16826 : RVal = Res;
680 16826 : RChain = Res.getValue(1);
681 126750 : }
682 126750 : break;
683 :
684 73720 : case TargetLowering::Promote: {
685 73720 : MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
686 : assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
687 : "Can only promote loads to same size type");
688 :
689 147440 : SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
690 147440 : RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
691 73720 : RChain = Res.getValue(1);
692 : break;
693 : }
694 : }
695 5111876 : if (RChain.getNode() != Node) {
696 : assert(RVal.getNode() != Node && "Load must be completely replaced");
697 167678 : DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
698 167678 : DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
699 83839 : if (UpdatedNodes) {
700 2784 : UpdatedNodes->insert(RVal.getNode());
701 2784 : UpdatedNodes->insert(RChain.getNode());
702 : }
703 83839 : ReplacedNode(Node);
704 : }
705 : return;
706 : }
707 :
708 : LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
709 100112 : EVT SrcVT = LD->getMemoryVT();
710 100112 : unsigned SrcWidth = SrcVT.getSizeInBits();
711 100112 : unsigned Alignment = LD->getAlignment();
712 100112 : MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
713 : AAMDNodes AAInfo = LD->getAAInfo();
714 :
715 100112 : if (SrcWidth != SrcVT.getStoreSizeInBits() &&
716 : // Some targets pretend to have an i1 loading operation, and actually
717 : // load an i8. This trick is correct for ZEXTLOAD because the top 7
718 : // bits are guaranteed to be zero; it helps the optimizers understand
719 : // that these bits are zero. It is also useful for EXTLOAD, since it
720 : // tells the optimizers that those bits are undefined. It would be
721 : // nice to have an effective generic way of getting these benefits...
722 : // Until such a way is found, don't insist on promoting i1 here.
723 1240 : (SrcVT != MVT::i1 ||
724 442 : TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
725 : TargetLowering::Promote)) {
726 : // Promote to a byte-sized load if not loading an integral number of
727 : // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
728 : unsigned NewWidth = SrcVT.getStoreSizeInBits();
729 531 : EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
730 : SDValue Ch;
731 :
732 : // The extra bits are guaranteed to be zero, since we stored them that
733 : // way. A zext load from NVT thus automatically gives zext from SrcVT.
734 :
735 : ISD::LoadExtType NewExtType =
736 531 : ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
737 :
738 : SDValue Result =
739 531 : DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
740 1593 : LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
741 :
742 531 : Ch = Result.getValue(1); // The chain.
743 :
744 531 : if (ExtType == ISD::SEXTLOAD)
745 : // Having the top bits zero doesn't help when sign extending.
746 75 : Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
747 : Result.getValueType(),
748 75 : Result, DAG.getValueType(SrcVT));
749 456 : else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
750 : // All the top bits are guaranteed to be zero - inform the optimizers.
751 240 : Result = DAG.getNode(ISD::AssertZext, dl,
752 : Result.getValueType(), Result,
753 240 : DAG.getValueType(SrcVT));
754 :
755 531 : Value = Result;
756 531 : Chain = Ch;
757 99581 : } else if (SrcWidth & (SrcWidth - 1)) {
758 : // If not loading a power-of-2 number of bits, expand as two loads.
759 : assert(!SrcVT.isVector() && "Unsupported extload!");
760 : unsigned RoundWidth = 1 << Log2_32(SrcWidth);
761 : assert(RoundWidth < SrcWidth);
762 288 : unsigned ExtraWidth = SrcWidth - RoundWidth;
763 : assert(ExtraWidth < RoundWidth);
764 : assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
765 : "Load size not an integral number of bytes!");
766 288 : EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
767 288 : EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
768 288 : SDValue Lo, Hi, Ch;
769 : unsigned IncrementSize;
770 288 : auto &DL = DAG.getDataLayout();
771 :
772 288 : if (DL.isLittleEndian()) {
773 : // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
774 : // Load the bottom RoundWidth bits.
775 524 : Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
776 262 : LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
777 786 : AAInfo);
778 :
779 : // Load the remaining ExtraWidth bits.
780 262 : IncrementSize = RoundWidth / 8;
781 262 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
782 : DAG.getConstant(IncrementSize, dl,
783 262 : Ptr.getValueType()));
784 262 : Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
785 262 : LD->getPointerInfo().getWithOffset(IncrementSize),
786 : ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
787 524 : AAInfo);
788 :
789 : // Build a factor node to remember that this load is independent of
790 : // the other one.
791 262 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
792 262 : Hi.getValue(1));
793 :
794 : // Move the top bits to the right place.
795 262 : Hi = DAG.getNode(
796 : ISD::SHL, dl, Hi.getValueType(), Hi,
797 : DAG.getConstant(RoundWidth, dl,
798 524 : TLI.getShiftAmountTy(Hi.getValueType(), DL)));
799 :
800 : // Join the hi and lo parts.
801 786 : Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
802 : } else {
803 : // Big endian - avoid unaligned loads.
804 : // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
805 : // Load the top RoundWidth bits.
806 52 : Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
807 26 : LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
808 78 : AAInfo);
809 :
810 : // Load the remaining ExtraWidth bits.
811 26 : IncrementSize = RoundWidth / 8;
812 26 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
813 : DAG.getConstant(IncrementSize, dl,
814 26 : Ptr.getValueType()));
815 26 : Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
816 26 : LD->getPointerInfo().getWithOffset(IncrementSize),
817 : ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
818 52 : AAInfo);
819 :
820 : // Build a factor node to remember that this load is independent of
821 : // the other one.
822 26 : Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
823 26 : Hi.getValue(1));
824 :
825 : // Move the top bits to the right place.
826 26 : Hi = DAG.getNode(
827 : ISD::SHL, dl, Hi.getValueType(), Hi,
828 : DAG.getConstant(ExtraWidth, dl,
829 52 : TLI.getShiftAmountTy(Hi.getValueType(), DL)));
830 :
831 : // Join the hi and lo parts.
832 78 : Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
833 : }
834 :
835 288 : Chain = Ch;
836 : } else {
837 : bool isCustom = false;
838 198586 : switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
839 : SrcVT.getSimpleVT())) {
840 0 : default: llvm_unreachable("This action is not supported yet!");
841 6383 : case TargetLowering::Custom:
842 : isCustom = true;
843 : LLVM_FALLTHROUGH;
844 98919 : case TargetLowering::Legal:
845 98919 : Value = SDValue(Node, 0);
846 98919 : Chain = SDValue(Node, 1);
847 :
848 98919 : if (isCustom) {
849 12766 : if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
850 4470 : Value = Res;
851 4470 : Chain = Res.getValue(1);
852 : }
853 : } else {
854 : // If this is an unaligned load and the target doesn't support it,
855 : // expand it.
856 92536 : EVT MemVT = LD->getMemoryVT();
857 : unsigned AS = LD->getAddressSpace();
858 92536 : unsigned Align = LD->getAlignment();
859 92536 : const DataLayout &DL = DAG.getDataLayout();
860 92536 : if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
861 484 : std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
862 : }
863 : }
864 : break;
865 :
866 374 : case TargetLowering::Expand: {
867 374 : EVT DestVT = Node->getValueType(0);
868 374 : if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
869 : // If the source type is not legal, see if there is a legal extload to
870 : // an intermediate type that we can then extend further.
871 : EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
872 56 : if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
873 : TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
874 : // If we are loading a legal type, this is a non-extload followed by a
875 : // full extend.
876 : ISD::LoadExtType MidExtType =
877 354 : (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
878 :
879 354 : SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
880 354 : SrcVT, LD->getMemOperand());
881 : unsigned ExtendOp =
882 354 : ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
883 1062 : Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
884 354 : Chain = Load.getValue(1);
885 : break;
886 : }
887 :
888 : // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
889 : // normal undefined upper bits behavior to allow using an in-reg extend
890 : // with the illegal FP type, so load as an integer and do the
891 : // from-integer conversion.
892 0 : if (SrcVT.getScalarType() == MVT::f16) {
893 0 : EVT ISrcVT = SrcVT.changeTypeToInteger();
894 0 : EVT IDestVT = DestVT.changeTypeToInteger();
895 0 : EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
896 :
897 0 : SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
898 : Chain, Ptr, ISrcVT,
899 0 : LD->getMemOperand());
900 0 : Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
901 0 : Chain = Result.getValue(1);
902 : break;
903 : }
904 : }
905 :
906 : assert(!SrcVT.isVector() &&
907 : "Vector Loads are handled in LegalizeVectorOps");
908 :
909 : // FIXME: This does not work for vectors on most targets. Sign-
910 : // and zero-extend operations are currently folded into extending
911 : // loads, whether they are legal or not, and then we end up here
912 : // without any support for legalizing them.
913 : assert(ExtType != ISD::EXTLOAD &&
914 : "EXTLOAD should always be supported!");
915 : // Turn the unsupported load into an EXTLOAD followed by an
916 : // explicit zero/sign extend inreg.
917 20 : SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
918 : Node->getValueType(0),
919 : Chain, Ptr, SrcVT,
920 40 : LD->getMemOperand());
921 : SDValue ValRes;
922 20 : if (ExtType == ISD::SEXTLOAD)
923 19 : ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
924 : Result.getValueType(),
925 19 : Result, DAG.getValueType(SrcVT));
926 : else
927 1 : ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
928 20 : Value = ValRes;
929 20 : Chain = Result.getValue(1);
930 20 : break;
931 : }
932 : }
933 : }
934 :
935 : // Since loads produce two values, make sure to remember that we legalized
936 : // both of them.
937 100112 : if (Chain.getNode() != Node) {
938 : assert(Value.getNode() != Node && "Load must be completely replaced");
939 12258 : DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
940 12258 : DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
941 6129 : if (UpdatedNodes) {
942 1 : UpdatedNodes->insert(Value.getNode());
943 1 : UpdatedNodes->insert(Chain.getNode());
944 : }
945 6129 : ReplacedNode(Node);
946 : }
947 : }
948 :
949 : /// Return a legal replacement for the given operation, with all legal operands.
950 78333261 : void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
951 : LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
952 :
953 : // Allow illegal target nodes and illegal registers.
954 156666522 : if (Node->getOpcode() == ISD::TargetConstant ||
955 : Node->getOpcode() == ISD::Register)
956 : return;
957 :
958 : #ifndef NDEBUG
959 : for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
960 : assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
961 : TargetLowering::TypeLegal ||
962 : TLI.isTypeLegal(Node->getValueType(i))) &&
963 : "Unexpected illegal type!");
964 :
965 : for (const SDValue &Op : Node->op_values())
966 : assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
967 : TargetLowering::TypeLegal ||
968 : TLI.isTypeLegal(Op.getValueType()) ||
969 : Op.getOpcode() == ISD::TargetConstant ||
970 : Op.getOpcode() == ISD::Register) &&
971 : "Unexpected illegal type!");
972 : #endif
973 :
974 : // Figure out the correct action; the way to query this varies by opcode
975 : TargetLowering::LegalizeAction Action = TargetLowering::Legal;
976 : bool SimpleFinishLegalizing = true;
977 66901850 : switch (Node->getOpcode()) {
978 85644 : case ISD::INTRINSIC_W_CHAIN:
979 : case ISD::INTRINSIC_WO_CHAIN:
980 : case ISD::INTRINSIC_VOID:
981 : case ISD::STACKSAVE:
982 85644 : Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
983 : break;
984 4 : case ISD::GET_DYNAMIC_AREA_OFFSET:
985 4 : Action = TLI.getOperationAction(Node->getOpcode(),
986 : Node->getValueType(0));
987 : break;
988 202 : case ISD::VAARG:
989 202 : Action = TLI.getOperationAction(Node->getOpcode(),
990 : Node->getValueType(0));
991 202 : if (Action != TargetLowering::Promote)
992 : Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
993 : break;
994 417276 : case ISD::FP_TO_FP16:
995 : case ISD::SINT_TO_FP:
996 : case ISD::UINT_TO_FP:
997 : case ISD::EXTRACT_VECTOR_ELT:
998 834552 : Action = TLI.getOperationAction(Node->getOpcode(),
999 417276 : Node->getOperand(0).getValueType());
1000 : break;
1001 29591 : case ISD::FP_ROUND_INREG:
1002 : case ISD::SIGN_EXTEND_INREG: {
1003 29591 : EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1004 29591 : Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1005 : break;
1006 : }
1007 2807 : case ISD::ATOMIC_STORE:
1008 5614 : Action = TLI.getOperationAction(Node->getOpcode(),
1009 2807 : Node->getOperand(2).getValueType());
1010 : break;
1011 83695 : case ISD::SELECT_CC:
1012 : case ISD::SETCC:
1013 : case ISD::BR_CC: {
1014 83695 : unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1015 : Node->getOpcode() == ISD::SETCC ? 2 : 1;
1016 83695 : unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1017 83695 : MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1018 : ISD::CondCode CCCode =
1019 83695 : cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1020 83695 : Action = TLI.getCondCodeAction(CCCode, OpVT);
1021 83695 : if (Action == TargetLowering::Legal) {
1022 82648 : if (Node->getOpcode() == ISD::SELECT_CC)
1023 21517 : Action = TLI.getOperationAction(Node->getOpcode(),
1024 : Node->getValueType(0));
1025 : else
1026 : Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1027 : }
1028 : break;
1029 : }
1030 : case ISD::LOAD:
1031 : case ISD::STORE:
1032 : // FIXME: Model these properly. LOAD and STORE are complicated, and
1033 : // STORE expects the unlegalized operand in some cases.
1034 : SimpleFinishLegalizing = false;
1035 : break;
1036 : case ISD::CALLSEQ_START:
1037 : case ISD::CALLSEQ_END:
1038 : // FIXME: This shouldn't be necessary. These nodes have special properties
1039 : // dealing with the recursive nature of legalization. Removing this
1040 : // special case should be done as part of making LegalizeDAG non-recursive.
1041 : SimpleFinishLegalizing = false;
1042 : break;
1043 231917 : case ISD::EXTRACT_ELEMENT:
1044 : case ISD::FLT_ROUNDS_:
1045 : case ISD::MERGE_VALUES:
1046 : case ISD::EH_RETURN:
1047 : case ISD::FRAME_TO_ARGS_OFFSET:
1048 : case ISD::EH_DWARF_CFA:
1049 : case ISD::EH_SJLJ_SETJMP:
1050 : case ISD::EH_SJLJ_LONGJMP:
1051 : case ISD::EH_SJLJ_SETUP_DISPATCH:
1052 : // These operations lie about being legal: when they claim to be legal,
1053 : // they should actually be expanded.
1054 231917 : Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1055 231917 : if (Action == TargetLowering::Legal)
1056 : Action = TargetLowering::Expand;
1057 : break;
1058 217 : case ISD::INIT_TRAMPOLINE:
1059 : case ISD::ADJUST_TRAMPOLINE:
1060 : case ISD::FRAMEADDR:
1061 : case ISD::RETURNADDR:
1062 : case ISD::ADDROFRETURNADDR:
1063 : // These operations lie about being legal: when they claim to be legal,
1064 : // they should actually be custom-lowered.
1065 217 : Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1066 217 : if (Action == TargetLowering::Legal)
1067 : Action = TargetLowering::Custom;
1068 : break;
1069 25 : case ISD::READCYCLECOUNTER:
1070 : // READCYCLECOUNTER returns an i64, even if type legalization might have
1071 : // expanded that to several smaller types.
1072 25 : Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1073 : break;
1074 : case ISD::READ_REGISTER:
1075 : case ISD::WRITE_REGISTER:
1076 : // Named register is legal in the DAG, but blocked by register name
1077 : // selection if not implemented by target (to chose the correct register)
1078 : // They'll be converted to Copy(To/From)Reg.
1079 : Action = TargetLowering::Legal;
1080 : break;
1081 30 : case ISD::DEBUGTRAP:
1082 30 : Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1083 30 : if (Action == TargetLowering::Expand) {
1084 : // replace ISD::DEBUGTRAP with ISD::TRAP
1085 : SDValue NewVal;
1086 13 : NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1087 26 : Node->getOperand(0));
1088 13 : ReplaceNode(Node, NewVal.getNode());
1089 13 : LegalizeOp(NewVal.getNode());
1090 : return;
1091 17 : }
1092 : break;
1093 593 : case ISD::STRICT_FADD:
1094 : case ISD::STRICT_FSUB:
1095 : case ISD::STRICT_FMUL:
1096 : case ISD::STRICT_FDIV:
1097 : case ISD::STRICT_FREM:
1098 : case ISD::STRICT_FSQRT:
1099 : case ISD::STRICT_FMA:
1100 : case ISD::STRICT_FPOW:
1101 : case ISD::STRICT_FPOWI:
1102 : case ISD::STRICT_FSIN:
1103 : case ISD::STRICT_FCOS:
1104 : case ISD::STRICT_FEXP:
1105 : case ISD::STRICT_FEXP2:
1106 : case ISD::STRICT_FLOG:
1107 : case ISD::STRICT_FLOG10:
1108 : case ISD::STRICT_FLOG2:
1109 : case ISD::STRICT_FRINT:
1110 : case ISD::STRICT_FNEARBYINT:
1111 : // These pseudo-ops get legalized as if they were their non-strict
1112 : // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1113 : // is also legal, but if ISD::FSQRT requires expansion then so does
1114 : // ISD::STRICT_FSQRT.
1115 593 : Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
1116 : Node->getValueType(0));
1117 : break;
1118 13 : case ISD::SADDSAT: {
1119 13 : Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1120 : break;
1121 : }
1122 113 : case ISD::MSCATTER:
1123 226 : Action = TLI.getOperationAction(Node->getOpcode(),
1124 : cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1125 : break;
1126 630 : case ISD::MSTORE:
1127 1260 : Action = TLI.getOperationAction(Node->getOpcode(),
1128 : cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1129 : break;
1130 50949266 : default:
1131 50949266 : if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1132 : Action = TargetLowering::Legal;
1133 : } else {
1134 45112085 : Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1135 : }
1136 : break;
1137 : }
1138 :
1139 : if (SimpleFinishLegalizing) {
1140 : SDNode *NewNode = Node;
1141 103605924 : switch (Node->getOpcode()) {
1142 : default: break;
1143 286500 : case ISD::SHL:
1144 : case ISD::SRL:
1145 : case ISD::SRA:
1146 : case ISD::ROTL:
1147 : case ISD::ROTR: {
1148 : // Legalizing shifts/rotates requires adjusting the shift amount
1149 : // to the appropriate width.
1150 286500 : SDValue Op0 = Node->getOperand(0);
1151 286500 : SDValue Op1 = Node->getOperand(1);
1152 573000 : if (!Op1.getValueType().isVector()) {
1153 563138 : SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1154 : // The getShiftAmountOperand() may create a new operand node or
1155 : // return the existing one. If new operand is created we need
1156 : // to update the parent node.
1157 : // Do not try to legalize SAO here! It will be automatically legalized
1158 : // in the next round.
1159 : if (SAO != Op1)
1160 26918 : NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1161 286500 : }
1162 : }
1163 286500 : break;
1164 910 : case ISD::SRL_PARTS:
1165 : case ISD::SRA_PARTS:
1166 : case ISD::SHL_PARTS: {
1167 : // Legalizing shifts/rotates requires adjusting the shift amount
1168 : // to the appropriate width.
1169 910 : SDValue Op0 = Node->getOperand(0);
1170 910 : SDValue Op1 = Node->getOperand(1);
1171 910 : SDValue Op2 = Node->getOperand(2);
1172 1820 : if (!Op2.getValueType().isVector()) {
1173 1820 : SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1174 : // The getShiftAmountOperand() may create a new operand node or
1175 : // return the existing one. If new operand is created we need
1176 : // to update the parent node.
1177 : if (SAO != Op2)
1178 0 : NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1179 : }
1180 : break;
1181 : }
1182 : }
1183 :
1184 51802962 : if (NewNode != Node) {
1185 122 : ReplaceNode(Node, NewNode);
1186 : Node = NewNode;
1187 : }
1188 51802963 : switch (Action) {
1189 : case TargetLowering::Legal:
1190 : LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1191 : return;
1192 2003832 : case TargetLowering::Custom:
1193 : LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1194 : // FIXME: The handling for custom lowering with multiple results is
1195 : // a complete mess.
1196 4007664 : if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1197 1934291 : if (!(Res.getNode() != Node || Res.getResNo() != 0))
1198 : return;
1199 :
1200 1340745 : if (Node->getNumValues() == 1) {
1201 : LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1202 : // We can just directly replace this node with the lowered value.
1203 1114920 : ReplaceNode(SDValue(Node, 0), Res);
1204 1114920 : return;
1205 : }
1206 :
1207 : SmallVector<SDValue, 8> ResultVals;
1208 677541 : for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1209 451716 : ResultVals.push_back(Res.getValue(i));
1210 : LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1211 225825 : ReplaceNode(Node, ResultVals.data());
1212 : return;
1213 2003829 : }
1214 : LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1215 : LLVM_FALLTHROUGH;
1216 : case TargetLowering::Expand:
1217 345107 : if (ExpandNode(Node))
1218 : return;
1219 : LLVM_FALLTHROUGH;
1220 : case TargetLowering::LibCall:
1221 48585 : ConvertNodeToLibcall(Node);
1222 48585 : return;
1223 16052 : case TargetLowering::Promote:
1224 16052 : PromoteNode(Node);
1225 16052 : return;
1226 : }
1227 : }
1228 :
1229 30197750 : switch (Node->getOpcode()) {
1230 0 : default:
1231 : #ifndef NDEBUG
1232 : dbgs() << "NODE: ";
1233 : Node->dump( &DAG);
1234 : dbgs() << "\n";
1235 : #endif
1236 0 : llvm_unreachable("Do not know how to legalize this operator!");
1237 :
1238 : case ISD::CALLSEQ_START:
1239 : case ISD::CALLSEQ_END:
1240 : break;
1241 5211989 : case ISD::LOAD:
1242 5211989 : return LegalizeLoadOps(Node);
1243 5859005 : case ISD::STORE:
1244 5859005 : return LegalizeStoreOps(Node);
1245 : }
1246 : }
1247 :
1248 0 : SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1249 0 : SDValue Vec = Op.getOperand(0);
1250 0 : SDValue Idx = Op.getOperand(1);
1251 0 : SDLoc dl(Op);
1252 :
1253 : // Before we generate a new store to a temporary stack slot, see if there is
1254 : // already one that we can use. There often is because when we scalarize
1255 : // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1256 : // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1257 : // the vector. If all are expanded here, we don't want one store per vector
1258 : // element.
1259 :
1260 : // Caches for hasPredecessorHelper
1261 : SmallPtrSet<const SDNode *, 32> Visited;
1262 : SmallVector<const SDNode *, 16> Worklist;
1263 0 : Visited.insert(Op.getNode());
1264 0 : Worklist.push_back(Idx.getNode());
1265 0 : SDValue StackPtr, Ch;
1266 0 : for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1267 0 : UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1268 : SDNode *User = *UI;
1269 : if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1270 0 : if (ST->isIndexed() || ST->isTruncatingStore() ||
1271 : ST->getValue() != Vec)
1272 0 : continue;
1273 :
1274 : // Make sure that nothing else could have stored into the destination of
1275 : // this store.
1276 0 : if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1277 0 : continue;
1278 :
1279 : // If the index is dependent on the store we will introduce a cycle when
1280 : // creating the load (the load uses the index, and by replacing the chain
1281 : // we will make the index dependent on the load). Also, the store might be
1282 : // dependent on the extractelement and introduce a cycle when creating
1283 : // the load.
1284 0 : if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1285 0 : ST->hasPredecessor(Op.getNode()))
1286 0 : continue;
1287 :
1288 0 : StackPtr = ST->getBasePtr();
1289 0 : Ch = SDValue(ST, 0);
1290 0 : break;
1291 : }
1292 : }
1293 :
1294 0 : EVT VecVT = Vec.getValueType();
1295 :
1296 0 : if (!Ch.getNode()) {
1297 : // Store the value to a temporary stack slot, then LOAD the returned part.
1298 0 : StackPtr = DAG.CreateStackTemporary(VecVT);
1299 0 : Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1300 0 : MachinePointerInfo());
1301 : }
1302 :
1303 0 : StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1304 :
1305 : SDValue NewLoad;
1306 :
1307 0 : if (Op.getValueType().isVector())
1308 0 : NewLoad =
1309 0 : DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1310 : else
1311 0 : NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1312 : MachinePointerInfo(),
1313 0 : VecVT.getVectorElementType());
1314 :
1315 : // Replace the chain going out of the store, by the one out of the load.
1316 0 : DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1317 :
1318 : // We introduced a cycle though, so update the loads operands, making sure
1319 : // to use the original store's chain as an incoming chain.
1320 : SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1321 0 : NewLoad->op_end());
1322 0 : NewLoadOperands[0] = Ch;
1323 : NewLoad =
1324 0 : SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1325 0 : return NewLoad;
1326 : }
1327 :
1328 0 : SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1329 : assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1330 :
1331 0 : SDValue Vec = Op.getOperand(0);
1332 0 : SDValue Part = Op.getOperand(1);
1333 0 : SDValue Idx = Op.getOperand(2);
1334 0 : SDLoc dl(Op);
1335 :
1336 : // Store the value to a temporary stack slot, then LOAD the returned part.
1337 0 : EVT VecVT = Vec.getValueType();
1338 0 : SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1339 0 : int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1340 : MachinePointerInfo PtrInfo =
1341 0 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1342 :
1343 : // First store the whole vector.
1344 0 : SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1345 :
1346 : // Then store the inserted part.
1347 0 : SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1348 :
1349 : // Store the subvector.
1350 0 : Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1351 :
1352 : // Finally, load the updated vector.
1353 0 : return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1354 : }
1355 :
1356 0 : SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1357 : // We can't handle this case efficiently. Allocate a sufficiently
1358 : // aligned object on the stack, store each element into it, then load
1359 : // the result as a vector.
1360 : // Create the stack frame object.
1361 0 : EVT VT = Node->getValueType(0);
1362 0 : EVT EltVT = VT.getVectorElementType();
1363 : SDLoc dl(Node);
1364 0 : SDValue FIPtr = DAG.CreateStackTemporary(VT);
1365 0 : int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1366 : MachinePointerInfo PtrInfo =
1367 0 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1368 :
1369 : // Emit a store of each element to the stack slot.
1370 : SmallVector<SDValue, 8> Stores;
1371 0 : unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1372 : // Store (in the right endianness) the elements to memory.
1373 0 : for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1374 : // Ignore undef elements.
1375 0 : if (Node->getOperand(i).isUndef()) continue;
1376 :
1377 0 : unsigned Offset = TypeByteSize*i;
1378 :
1379 0 : SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1380 0 : Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1381 :
1382 : // If the destination vector element type is narrower than the source
1383 : // element type, only store the bits necessary.
1384 0 : if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1385 0 : Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1386 0 : Node->getOperand(i), Idx,
1387 0 : PtrInfo.getWithOffset(Offset), EltVT));
1388 : } else
1389 0 : Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1390 0 : Idx, PtrInfo.getWithOffset(Offset)));
1391 : }
1392 :
1393 0 : SDValue StoreChain;
1394 0 : if (!Stores.empty()) // Not all undef elements?
1395 0 : StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1396 : else
1397 0 : StoreChain = DAG.getEntryNode();
1398 :
1399 : // Result is a load from the stack slot.
1400 0 : return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1401 : }
1402 :
1403 : /// Bitcast a floating-point value to an integer value. Only bitcast the part
1404 : /// containing the sign bit if the target has no integer value capable of
1405 : /// holding all bits of the floating-point value.
1406 0 : void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1407 : const SDLoc &DL,
1408 : SDValue Value) const {
1409 0 : EVT FloatVT = Value.getValueType();
1410 0 : unsigned NumBits = FloatVT.getSizeInBits();
1411 0 : State.FloatVT = FloatVT;
1412 0 : EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1413 : // Convert to an integer of the same size.
1414 0 : if (TLI.isTypeLegal(IVT)) {
1415 0 : State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1416 0 : State.SignMask = APInt::getSignMask(NumBits);
1417 0 : State.SignBit = NumBits - 1;
1418 0 : return;
1419 : }
1420 :
1421 0 : auto &DataLayout = DAG.getDataLayout();
1422 : // Store the float to memory, then load the sign part out as an integer.
1423 0 : MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1424 : // First create a temporary that is aligned for both the load and store.
1425 0 : SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1426 0 : int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1427 : // Then store the float to it.
1428 0 : State.FloatPtr = StackPtr;
1429 0 : MachineFunction &MF = DAG.getMachineFunction();
1430 0 : State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1431 0 : State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1432 0 : State.FloatPointerInfo);
1433 :
1434 : SDValue IntPtr;
1435 0 : if (DataLayout.isBigEndian()) {
1436 : assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1437 : // Load out a legal integer with the same sign bit as the float.
1438 : IntPtr = StackPtr;
1439 0 : State.IntPointerInfo = State.FloatPointerInfo;
1440 : } else {
1441 : // Advance the pointer so that the loaded byte will contain the sign bit.
1442 0 : unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1443 0 : IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1444 0 : DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1445 : State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1446 0 : ByteOffset);
1447 : }
1448 :
1449 0 : State.IntPtr = IntPtr;
1450 0 : State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1451 0 : State.IntPointerInfo, MVT::i8);
1452 0 : State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1453 0 : State.SignBit = 7;
1454 : }
1455 :
1456 : /// Replace the integer value produced by getSignAsIntValue() with a new value
1457 : /// and cast the result back to a floating-point type.
1458 0 : SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1459 : const SDLoc &DL,
1460 : SDValue NewIntValue) const {
1461 0 : if (!State.Chain)
1462 0 : return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1463 :
1464 : // Override the part containing the sign bit in the value stored on the stack.
1465 0 : SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1466 0 : State.IntPointerInfo, MVT::i8);
1467 0 : return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1468 0 : State.FloatPointerInfo);
1469 : }
1470 :
1471 200 : SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1472 : SDLoc DL(Node);
1473 200 : SDValue Mag = Node->getOperand(0);
1474 200 : SDValue Sign = Node->getOperand(1);
1475 :
1476 : // Get sign bit into an integer value.
1477 : FloatSignAsInt SignAsInt;
1478 200 : getSignAsIntValue(SignAsInt, DL, Sign);
1479 :
1480 200 : EVT IntVT = SignAsInt.IntValue.getValueType();
1481 200 : SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1482 200 : SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1483 200 : SignMask);
1484 :
1485 : // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1486 200 : EVT FloatVT = Mag.getValueType();
1487 200 : if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1488 : TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1489 316 : SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1490 316 : SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1491 158 : SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1492 158 : DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1493 158 : return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1494 : }
1495 :
1496 : // Transform Mag value to integer, and clear the sign bit.
1497 : FloatSignAsInt MagAsInt;
1498 42 : getSignAsIntValue(MagAsInt, DL, Mag);
1499 42 : EVT MagVT = MagAsInt.IntValue.getValueType();
1500 126 : SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1501 42 : SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1502 42 : ClearSignMask);
1503 :
1504 : // Get the signbit at the right position for MagAsInt.
1505 42 : int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1506 42 : EVT ShiftVT = IntVT;
1507 42 : if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1508 0 : SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1509 0 : ShiftVT = MagVT;
1510 : }
1511 42 : if (ShiftAmount > 0) {
1512 22 : SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1513 44 : SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1514 20 : } else if (ShiftAmount < 0) {
1515 0 : SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1516 0 : SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1517 : }
1518 42 : if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1519 42 : SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1520 : }
1521 :
1522 : // Store the part with the modified sign and convert back to float.
1523 84 : SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1524 42 : return modifySignAsInt(MagAsInt, DL, CopiedSign);
1525 : }
1526 :
1527 2 : SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1528 : SDLoc DL(Node);
1529 2 : SDValue Value = Node->getOperand(0);
1530 :
1531 : // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1532 2 : EVT FloatVT = Value.getValueType();
1533 2 : if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1534 2 : SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1535 4 : return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1536 : }
1537 :
1538 : // Transform value to integer, clear the sign bit and transform back.
1539 : FloatSignAsInt ValueAsInt;
1540 0 : getSignAsIntValue(ValueAsInt, DL, Value);
1541 0 : EVT IntVT = ValueAsInt.IntValue.getValueType();
1542 0 : SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1543 0 : SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1544 0 : ClearSignMask);
1545 0 : return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1546 : }
1547 :
1548 0 : void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1549 : SmallVectorImpl<SDValue> &Results) {
1550 0 : unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1551 : assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1552 : " not tell us which reg is the stack pointer!");
1553 : SDLoc dl(Node);
1554 0 : EVT VT = Node->getValueType(0);
1555 : SDValue Tmp1 = SDValue(Node, 0);
1556 : SDValue Tmp2 = SDValue(Node, 1);
1557 0 : SDValue Tmp3 = Node->getOperand(2);
1558 0 : SDValue Chain = Tmp1.getOperand(0);
1559 :
1560 : // Chain the dynamic stack allocation so that it doesn't modify the stack
1561 : // pointer when other instructions are using the stack.
1562 0 : Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1563 :
1564 0 : SDValue Size = Tmp2.getOperand(1);
1565 0 : SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1566 0 : Chain = SP.getValue(1);
1567 0 : unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1568 : unsigned StackAlign =
1569 0 : DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1570 0 : Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1571 0 : if (Align > StackAlign)
1572 0 : Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1573 0 : DAG.getConstant(-(uint64_t)Align, dl, VT));
1574 0 : Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1575 :
1576 0 : Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1577 0 : DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1578 :
1579 0 : Results.push_back(Tmp1);
1580 0 : Results.push_back(Tmp2);
1581 0 : }
1582 :
1583 : /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1584 : /// target.
1585 : ///
1586 : /// If the SETCC has been legalized using AND / OR, then the legalized node
1587 : /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1588 : /// will be set to false.
1589 : ///
1590 : /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1591 : /// then the values of LHS and RHS will be swapped, CC will be set to the
1592 : /// new condition, and NeedInvert will be set to false.
1593 : ///
1594 : /// If the SETCC has been legalized using the inverse condcode, then LHS and
1595 : /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1596 : /// will be set to true. The caller must invert the result of the SETCC with
1597 : /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1598 : /// of a true/false result.
1599 : ///
1600 : /// \returns true if the SetCC has been legalized, false if it hasn't.
1601 0 : bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1602 : SDValue &RHS, SDValue &CC,
1603 : bool &NeedInvert,
1604 : const SDLoc &dl) {
1605 : MVT OpVT = LHS.getSimpleValueType();
1606 0 : ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1607 0 : NeedInvert = false;
1608 : bool NeedSwap = false;
1609 0 : switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1610 0 : default: llvm_unreachable("Unknown condition code action!");
1611 : case TargetLowering::Legal:
1612 : // Nothing to do.
1613 : break;
1614 0 : case TargetLowering::Expand: {
1615 0 : ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1616 0 : if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1617 : std::swap(LHS, RHS);
1618 0 : CC = DAG.getCondCode(InvCC);
1619 0 : return true;
1620 : }
1621 : // Swapping operands didn't work. Try inverting the condition.
1622 0 : InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1623 0 : if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1624 : // If inverting the condition is not enough, try swapping operands
1625 : // on top of it.
1626 0 : InvCC = ISD::getSetCCSwappedOperands(InvCC);
1627 : NeedSwap = true;
1628 : }
1629 0 : if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1630 0 : CC = DAG.getCondCode(InvCC);
1631 0 : NeedInvert = true;
1632 0 : if (NeedSwap)
1633 : std::swap(LHS, RHS);
1634 0 : return true;
1635 : }
1636 :
1637 : ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1638 : unsigned Opc = 0;
1639 : switch (CCCode) {
1640 0 : default: llvm_unreachable("Don't know how to expand this condition!");
1641 : case ISD::SETO:
1642 : assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)
1643 : && "If SETO is expanded, SETOEQ must be legal!");
1644 : CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1645 0 : case ISD::SETUO:
1646 : assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)
1647 : && "If SETUO is expanded, SETUNE must be legal!");
1648 0 : CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1649 0 : case ISD::SETOEQ:
1650 : case ISD::SETOGT:
1651 : case ISD::SETOGE:
1652 : case ISD::SETOLT:
1653 : case ISD::SETOLE:
1654 : case ISD::SETONE:
1655 : case ISD::SETUEQ:
1656 : case ISD::SETUNE:
1657 : case ISD::SETUGT:
1658 : case ISD::SETUGE:
1659 : case ISD::SETULT:
1660 : case ISD::SETULE:
1661 : // If we are floating point, assign and break, otherwise fall through.
1662 0 : if (!OpVT.isInteger()) {
1663 : // We can use the 4th bit to tell if we are the unordered
1664 : // or ordered version of the opcode.
1665 0 : CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1666 0 : Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1667 0 : CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1668 0 : break;
1669 : }
1670 : // Fallthrough if we are unsigned integer.
1671 : LLVM_FALLTHROUGH;
1672 : case ISD::SETLE:
1673 : case ISD::SETGT:
1674 : case ISD::SETGE:
1675 : case ISD::SETLT:
1676 : case ISD::SETNE:
1677 : case ISD::SETEQ:
1678 : // If all combinations of inverting the condition and swapping operands
1679 : // didn't work then we have no means to expand the condition.
1680 0 : llvm_unreachable("Don't know how to expand this condition!");
1681 : }
1682 :
1683 0 : SDValue SetCC1, SetCC2;
1684 0 : if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1685 : // If we aren't the ordered or unorder operation,
1686 : // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1687 0 : SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1688 0 : SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1689 : } else {
1690 : // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1691 0 : SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1692 0 : SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1693 : }
1694 0 : LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1695 0 : RHS = SDValue();
1696 0 : CC = SDValue();
1697 0 : return true;
1698 : }
1699 : }
1700 : return false;
1701 : }
1702 :
1703 : /// Emit a store/load combination to the stack. This stores
1704 : /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1705 : /// a load from the stack slot to DestVT, extending it if needed.
1706 : /// The resultant code need not be legal.
1707 0 : SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1708 : EVT DestVT, const SDLoc &dl) {
1709 : // Create the stack frame object.
1710 0 : unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1711 0 : SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1712 0 : SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1713 :
1714 : FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1715 0 : int SPFI = StackPtrFI->getIndex();
1716 : MachinePointerInfo PtrInfo =
1717 0 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1718 :
1719 0 : unsigned SrcSize = SrcOp.getValueSizeInBits();
1720 0 : unsigned SlotSize = SlotVT.getSizeInBits();
1721 0 : unsigned DestSize = DestVT.getSizeInBits();
1722 0 : Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1723 0 : unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1724 :
1725 : // Emit a store to the stack slot. Use a truncstore if the input value is
1726 : // later than DestVT.
1727 0 : SDValue Store;
1728 :
1729 0 : if (SrcSize > SlotSize)
1730 0 : Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1731 0 : SlotVT, SrcAlign);
1732 : else {
1733 : assert(SrcSize == SlotSize && "Invalid store");
1734 0 : Store =
1735 0 : DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1736 : }
1737 :
1738 : // Result is a load from the stack slot.
1739 0 : if (SlotSize == DestSize)
1740 0 : return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1741 :
1742 : assert(SlotSize < DestSize && "Unknown extension!");
1743 0 : return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1744 0 : DestAlign);
1745 : }
1746 :
1747 0 : SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1748 : SDLoc dl(Node);
1749 : // Create a vector sized/aligned stack slot, store the value to element #0,
1750 : // then load the whole vector back out.
1751 0 : SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1752 :
1753 : FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1754 0 : int SPFI = StackPtrFI->getIndex();
1755 :
1756 0 : SDValue Ch = DAG.getTruncStore(
1757 0 : DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1758 0 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1759 0 : Node->getValueType(0).getVectorElementType());
1760 0 : return DAG.getLoad(
1761 : Node->getValueType(0), dl, Ch, StackPtr,
1762 0 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1763 : }
1764 :
1765 : static bool
1766 27 : ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1767 : const TargetLowering &TLI, SDValue &Res) {
1768 27 : unsigned NumElems = Node->getNumOperands();
1769 : SDLoc dl(Node);
1770 54 : EVT VT = Node->getValueType(0);
1771 :
1772 : // Try to group the scalars into pairs, shuffle the pairs together, then
1773 : // shuffle the pairs of pairs together, etc. until the vector has
1774 : // been built. This will work only if all of the necessary shuffle masks
1775 : // are legal.
1776 :
1777 : // We do this in two phases; first to check the legality of the shuffles,
1778 : // and next, assuming that all shuffles are legal, to create the new nodes.
1779 81 : for (int Phase = 0; Phase < 2; ++Phase) {
1780 54 : SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1781 54 : NewIntermedVals;
1782 398 : for (unsigned i = 0; i < NumElems; ++i) {
1783 688 : SDValue V = Node->getOperand(i);
1784 344 : if (V.isUndef())
1785 16 : continue;
1786 :
1787 : SDValue Vec;
1788 328 : if (Phase)
1789 164 : Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1790 984 : IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1791 : }
1792 :
1793 184 : while (IntermedVals.size() > 2) {
1794 76 : NewIntermedVals.clear();
1795 296 : for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1796 : // This vector and the next vector are shuffled together (simply to
1797 : // append the one to the other).
1798 220 : SmallVector<int, 16> ShuffleVec(NumElems, -1);
1799 :
1800 : SmallVector<int, 16> FinalIndices;
1801 660 : FinalIndices.reserve(IntermedVals[i].second.size() +
1802 440 : IntermedVals[i+1].second.size());
1803 :
1804 : int k = 0;
1805 520 : for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1806 : ++j, ++k) {
1807 300 : ShuffleVec[k] = j;
1808 600 : FinalIndices.push_back(IntermedVals[i].second[j]);
1809 : }
1810 520 : for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1811 : ++j, ++k) {
1812 300 : ShuffleVec[k] = NumElems + j;
1813 600 : FinalIndices.push_back(IntermedVals[i+1].second[j]);
1814 : }
1815 :
1816 : SDValue Shuffle;
1817 220 : if (Phase)
1818 110 : Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1819 : IntermedVals[i+1].first,
1820 110 : ShuffleVec);
1821 220 : else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1822 : return false;
1823 220 : NewIntermedVals.push_back(
1824 220 : std::make_pair(Shuffle, std::move(FinalIndices)));
1825 : }
1826 :
1827 : // If we had an odd number of defined values, then append the last
1828 : // element to the array of new vectors.
1829 152 : if ((IntermedVals.size() & 1) != 0)
1830 0 : NewIntermedVals.push_back(IntermedVals.back());
1831 :
1832 76 : IntermedVals.swap(NewIntermedVals);
1833 : }
1834 :
1835 : assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1836 : "Invalid number of intermediate vectors");
1837 54 : SDValue Vec1 = IntermedVals[0].first;
1838 54 : SDValue Vec2;
1839 54 : if (IntermedVals.size() > 1)
1840 54 : Vec2 = IntermedVals[1].first;
1841 0 : else if (Phase)
1842 0 : Vec2 = DAG.getUNDEF(VT);
1843 :
1844 54 : SmallVector<int, 16> ShuffleVec(NumElems, -1);
1845 218 : for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1846 492 : ShuffleVec[IntermedVals[0].second[i]] = i;
1847 218 : for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1848 492 : ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1849 :
1850 54 : if (Phase)
1851 27 : Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1852 54 : else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1853 : return false;
1854 : }
1855 :
1856 : return true;
1857 : }
1858 :
1859 : /// Expand a BUILD_VECTOR node on targets that don't
1860 : /// support the operation, but do support the resultant vector type.
1861 26205 : SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1862 26205 : unsigned NumElems = Node->getNumOperands();
1863 : SDValue Value1, Value2;
1864 : SDLoc dl(Node);
1865 26205 : EVT VT = Node->getValueType(0);
1866 26205 : EVT OpVT = Node->getOperand(0).getValueType();
1867 26205 : EVT EltVT = VT.getVectorElementType();
1868 :
1869 : // If the only non-undef value is the low element, turn this into a
1870 : // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1871 : bool isOnlyLowElement = true;
1872 : bool MoreThanTwoValues = false;
1873 : bool isConstant = true;
1874 302969 : for (unsigned i = 0; i < NumElems; ++i) {
1875 553528 : SDValue V = Node->getOperand(i);
1876 276764 : if (V.isUndef())
1877 14609 : continue;
1878 262155 : if (i > 0)
1879 : isOnlyLowElement = false;
1880 : if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1881 : isConstant = false;
1882 :
1883 262155 : if (!Value1.getNode()) {
1884 26205 : Value1 = V;
1885 235950 : } else if (!Value2.getNode()) {
1886 : if (V != Value1)
1887 16818 : Value2 = V;
1888 : } else if (V != Value1 && V != Value2) {
1889 : MoreThanTwoValues = true;
1890 : }
1891 : }
1892 :
1893 26205 : if (!Value1.getNode())
1894 0 : return DAG.getUNDEF(VT);
1895 :
1896 26205 : if (isOnlyLowElement)
1897 27 : return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1898 :
1899 : // If all elements are constants, create a load from the constant pool.
1900 26196 : if (isConstant) {
1901 : SmallVector<Constant*, 16> CV;
1902 290049 : for (unsigned i = 0, e = NumElems; i != e; ++i) {
1903 : if (ConstantFPSDNode *V =
1904 266136 : dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1905 12536 : CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1906 : } else if (ConstantSDNode *V =
1907 : dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1908 239516 : if (OpVT==EltVT)
1909 235816 : CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1910 : else {
1911 : // If OpVT and EltVT don't match, EltVT is not legal and the
1912 : // element values have been promoted/truncated earlier. Undo this;
1913 : // we don't want a v16i8 to become a v16i32 for example.
1914 3700 : const ConstantInt *CI = V->getConstantIntValue();
1915 3700 : CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1916 : CI->getZExtValue()));
1917 : }
1918 : } else {
1919 : assert(Node->getOperand(i).isUndef());
1920 14084 : Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1921 14084 : CV.push_back(UndefValue::get(OpNTy));
1922 : }
1923 : }
1924 23913 : Constant *CP = ConstantVector::get(CV);
1925 : SDValue CPIdx =
1926 47826 : DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1927 23913 : unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1928 23913 : return DAG.getLoad(
1929 23913 : VT, dl, DAG.getEntryNode(), CPIdx,
1930 : MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1931 23913 : Alignment);
1932 : }
1933 :
1934 2283 : SmallSet<SDValue, 16> DefinedValues;
1935 12891 : for (unsigned i = 0; i < NumElems; ++i) {
1936 31824 : if (Node->getOperand(i).isUndef())
1937 : continue;
1938 10094 : DefinedValues.insert(Node->getOperand(i));
1939 : }
1940 :
1941 4566 : if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1942 2226 : if (!MoreThanTwoValues) {
1943 2199 : SmallVector<int, 8> ShuffleVec(NumElems, -1);
1944 12459 : for (unsigned i = 0; i < NumElems; ++i) {
1945 20520 : SDValue V = Node->getOperand(i);
1946 10260 : if (V.isUndef())
1947 : continue;
1948 9755 : ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1949 : }
1950 4398 : if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1951 : // Get the splatted value into the low element of a vector register.
1952 4186 : SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1953 2093 : SDValue Vec2;
1954 2093 : if (Value2.getNode())
1955 2830 : Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1956 : else
1957 678 : Vec2 = DAG.getUNDEF(VT);
1958 :
1959 : // Return shuffle(LowValVec, undef, <0,0,0,0>)
1960 4186 : return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1961 : }
1962 : } else {
1963 27 : SDValue Res;
1964 27 : if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1965 27 : return Res;
1966 : }
1967 : }
1968 :
1969 : // Otherwise, we can't handle this case efficiently.
1970 163 : return ExpandVectorBuildThroughStack(Node);
1971 : }
1972 :
1973 : // Expand a node into a call to a libcall. If the result value
1974 : // does not fit into a register, return the lo part and set the hi part to the
1975 : // by-reg argument. If it does fit into a single register, return the result
1976 : // and leave the Hi part unset.
1977 5087 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1978 : bool isSigned) {
1979 : TargetLowering::ArgListTy Args;
1980 : TargetLowering::ArgListEntry Entry;
1981 11616 : for (const SDValue &Op : Node->op_values()) {
1982 6529 : EVT ArgVT = Op.getValueType();
1983 6529 : Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1984 6529 : Entry.Node = Op;
1985 6529 : Entry.Ty = ArgTy;
1986 6529 : Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1987 6529 : Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1988 6529 : Args.push_back(Entry);
1989 : }
1990 5087 : SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1991 10174 : TLI.getPointerTy(DAG.getDataLayout()));
1992 :
1993 5087 : EVT RetVT = Node->getValueType(0);
1994 5087 : Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
1995 :
1996 : // By default, the input chain to this libcall is the entry node of the
1997 : // function. If the libcall is going to be emitted as a tail call then
1998 : // TLI.isUsedByReturnOnly will change it to the right chain if the return
1999 : // node which is being folded has a non-entry input chain.
2000 5087 : SDValue InChain = DAG.getEntryNode();
2001 :
2002 : // isTailCall may be true since the callee does not reference caller stack
2003 : // frame. Check if it's in the right position and that the return types match.
2004 5087 : SDValue TCChain = InChain;
2005 5087 : const Function &F = DAG.getMachineFunction().getFunction();
2006 : bool isTailCall =
2007 5087 : TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2008 30 : (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2009 : if (isTailCall)
2010 391 : InChain = TCChain;
2011 :
2012 10174 : TargetLowering::CallLoweringInfo CLI(DAG);
2013 5087 : bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2014 5087 : CLI.setDebugLoc(SDLoc(Node))
2015 5087 : .setChain(InChain)
2016 5087 : .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2017 5087 : std::move(Args))
2018 : .setTailCall(isTailCall)
2019 : .setSExtResult(signExtend)
2020 5087 : .setZExtResult(!signExtend)
2021 : .setIsPostTypeLegalization(true);
2022 :
2023 5087 : std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2024 :
2025 5087 : if (!CallInfo.second.getNode()) {
2026 : LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump());
2027 : // It's a tailcall, return the chain (which is the DAG root).
2028 353 : return DAG.getRoot();
2029 : }
2030 :
2031 : LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump());
2032 4734 : return CallInfo.first;
2033 : }
2034 :
2035 : /// Generate a libcall taking the given operands as arguments
2036 : /// and returning a result of type RetVT.
2037 16 : SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2038 : const SDValue *Ops, unsigned NumOps,
2039 : bool isSigned, const SDLoc &dl) {
2040 : TargetLowering::ArgListTy Args;
2041 16 : Args.reserve(NumOps);
2042 :
2043 : TargetLowering::ArgListEntry Entry;
2044 80 : for (unsigned i = 0; i != NumOps; ++i) {
2045 64 : Entry.Node = Ops[i];
2046 128 : Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2047 64 : Entry.IsSExt = isSigned;
2048 64 : Entry.IsZExt = !isSigned;
2049 64 : Args.push_back(Entry);
2050 : }
2051 16 : SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2052 32 : TLI.getPointerTy(DAG.getDataLayout()));
2053 :
2054 16 : Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2055 :
2056 16 : TargetLowering::CallLoweringInfo CLI(DAG);
2057 : CLI.setDebugLoc(dl)
2058 16 : .setChain(DAG.getEntryNode())
2059 16 : .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2060 16 : std::move(Args))
2061 : .setSExtResult(isSigned)
2062 16 : .setZExtResult(!isSigned)
2063 : .setIsPostTypeLegalization(true);
2064 :
2065 16 : std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2066 :
2067 16 : return CallInfo.first;
2068 : }
2069 :
2070 : // Expand a node into a call to a libcall. Similar to
2071 : // ExpandLibCall except that the first operand is the in-chain.
2072 : std::pair<SDValue, SDValue>
2073 487 : SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2074 : SDNode *Node,
2075 : bool isSigned) {
2076 487 : SDValue InChain = Node->getOperand(0);
2077 :
2078 : TargetLowering::ArgListTy Args;
2079 : TargetLowering::ArgListEntry Entry;
2080 1480 : for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2081 1986 : EVT ArgVT = Node->getOperand(i).getValueType();
2082 993 : Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2083 993 : Entry.Node = Node->getOperand(i);
2084 993 : Entry.Ty = ArgTy;
2085 993 : Entry.IsSExt = isSigned;
2086 993 : Entry.IsZExt = !isSigned;
2087 993 : Args.push_back(Entry);
2088 : }
2089 487 : SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2090 974 : TLI.getPointerTy(DAG.getDataLayout()));
2091 :
2092 974 : Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2093 :
2094 974 : TargetLowering::CallLoweringInfo CLI(DAG);
2095 487 : CLI.setDebugLoc(SDLoc(Node))
2096 487 : .setChain(InChain)
2097 487 : .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2098 487 : std::move(Args))
2099 : .setSExtResult(isSigned)
2100 487 : .setZExtResult(!isSigned);
2101 :
2102 487 : std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2103 :
2104 487 : return CallInfo;
2105 : }
2106 :
2107 4137 : SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2108 : RTLIB::Libcall Call_F32,
2109 : RTLIB::Libcall Call_F64,
2110 : RTLIB::Libcall Call_F80,
2111 : RTLIB::Libcall Call_F128,
2112 : RTLIB::Libcall Call_PPCF128) {
2113 4137 : if (Node->isStrictFPOpcode())
2114 335 : Node = DAG.mutateStrictFPToFP(Node);
2115 :
2116 : RTLIB::Libcall LC;
2117 4137 : switch (Node->getSimpleValueType(0).SimpleTy) {
2118 0 : default: llvm_unreachable("Unexpected request for libcall!");
2119 : case MVT::f32: LC = Call_F32; break;
2120 984 : case MVT::f64: LC = Call_F64; break;
2121 73 : case MVT::f80: LC = Call_F80; break;
2122 58 : case MVT::f128: LC = Call_F128; break;
2123 0 : case MVT::ppcf128: LC = Call_PPCF128; break;
2124 : }
2125 4137 : return ExpandLibCall(LC, Node, false);
2126 : }
2127 :
2128 122 : SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2129 : RTLIB::Libcall Call_I8,
2130 : RTLIB::Libcall Call_I16,
2131 : RTLIB::Libcall Call_I32,
2132 : RTLIB::Libcall Call_I64,
2133 : RTLIB::Libcall Call_I128) {
2134 : RTLIB::Libcall LC;
2135 122 : switch (Node->getSimpleValueType(0).SimpleTy) {
2136 0 : default: llvm_unreachable("Unexpected request for libcall!");
2137 : case MVT::i8: LC = Call_I8; break;
2138 13 : case MVT::i16: LC = Call_I16; break;
2139 109 : case MVT::i32: LC = Call_I32; break;
2140 0 : case MVT::i64: LC = Call_I64; break;
2141 0 : case MVT::i128: LC = Call_I128; break;
2142 : }
2143 122 : return ExpandLibCall(LC, Node, isSigned);
2144 : }
2145 :
2146 : /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2147 : void
2148 0 : SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2149 : SmallVectorImpl<SDValue> &Results) {
2150 0 : unsigned Opcode = Node->getOpcode();
2151 0 : bool isSigned = Opcode == ISD::SDIVREM;
2152 :
2153 : RTLIB::Libcall LC;
2154 0 : switch (Node->getSimpleValueType(0).SimpleTy) {
2155 0 : default: llvm_unreachable("Unexpected request for libcall!");
2156 0 : case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2157 0 : case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2158 0 : case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2159 0 : case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2160 0 : case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2161 : }
2162 :
2163 : // The input chain to this libcall is the entry node of the function.
2164 : // Legalizing the call will automatically add the previous call to the
2165 : // dependence.
2166 0 : SDValue InChain = DAG.getEntryNode();
2167 :
2168 0 : EVT RetVT = Node->getValueType(0);
2169 0 : Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2170 :
2171 : TargetLowering::ArgListTy Args;
2172 : TargetLowering::ArgListEntry Entry;
2173 0 : for (const SDValue &Op : Node->op_values()) {
2174 0 : EVT ArgVT = Op.getValueType();
2175 0 : Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2176 0 : Entry.Node = Op;
2177 0 : Entry.Ty = ArgTy;
2178 0 : Entry.IsSExt = isSigned;
2179 0 : Entry.IsZExt = !isSigned;
2180 0 : Args.push_back(Entry);
2181 : }
2182 :
2183 : // Also pass the return address of the remainder.
2184 0 : SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2185 0 : Entry.Node = FIPtr;
2186 0 : Entry.Ty = RetTy->getPointerTo();
2187 0 : Entry.IsSExt = isSigned;
2188 0 : Entry.IsZExt = !isSigned;
2189 0 : Args.push_back(Entry);
2190 :
2191 0 : SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2192 0 : TLI.getPointerTy(DAG.getDataLayout()));
2193 :
2194 : SDLoc dl(Node);
2195 0 : TargetLowering::CallLoweringInfo CLI(DAG);
2196 : CLI.setDebugLoc(dl)
2197 0 : .setChain(InChain)
2198 0 : .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2199 0 : std::move(Args))
2200 : .setSExtResult(isSigned)
2201 : .setZExtResult(!isSigned);
2202 :
2203 0 : std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2204 :
2205 : // Remainder is loaded back from the stack frame.
2206 : SDValue Rem =
2207 0 : DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2208 0 : Results.push_back(CallInfo.first);
2209 0 : Results.push_back(Rem);
2210 0 : }
2211 :
2212 : /// Return true if sincos libcall is available.
2213 : static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2214 : RTLIB::Libcall LC;
2215 : switch (Node->getSimpleValueType(0).SimpleTy) {
2216 0 : default: llvm_unreachable("Unexpected request for libcall!");
2217 : case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2218 : case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2219 : case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2220 : case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2221 : case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2222 : }
2223 : return TLI.getLibcallName(LC) != nullptr;
2224 : }
2225 :
2226 : /// Only issue sincos libcall if both sin and cos are needed.
2227 : static bool useSinCos(SDNode *Node) {
2228 : unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2229 159 : ? ISD::FCOS : ISD::FSIN;
2230 :
2231 159 : SDValue Op0 = Node->getOperand(0);
2232 159 : for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2233 230 : UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2234 : SDNode *User = *UI;
2235 159 : if (User == Node)
2236 : continue;
2237 : // The other user might have been turned into sincos already.
2238 176 : if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2239 : return true;
2240 : }
2241 : return false;
2242 : }
2243 :
2244 : /// Issue libcalls to sincos to compute sin / cos pairs.
2245 : void
2246 0 : SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2247 : SmallVectorImpl<SDValue> &Results) {
2248 : RTLIB::Libcall LC;
2249 0 : switch (Node->getSimpleValueType(0).SimpleTy) {
2250 0 : default: llvm_unreachable("Unexpected request for libcall!");
2251 : case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2252 0 : case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2253 0 : case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2254 0 : case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2255 0 : case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2256 : }
2257 :
2258 : // The input chain to this libcall is the entry node of the function.
2259 : // Legalizing the call will automatically add the previous call to the
2260 : // dependence.
2261 0 : SDValue InChain = DAG.getEntryNode();
2262 :
2263 0 : EVT RetVT = Node->getValueType(0);
2264 0 : Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2265 :
2266 : TargetLowering::ArgListTy Args;
2267 : TargetLowering::ArgListEntry Entry;
2268 :
2269 : // Pass the argument.
2270 0 : Entry.Node = Node->getOperand(0);
2271 0 : Entry.Ty = RetTy;
2272 : Entry.IsSExt = false;
2273 : Entry.IsZExt = false;
2274 0 : Args.push_back(Entry);
2275 :
2276 : // Pass the return address of sin.
2277 0 : SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2278 0 : Entry.Node = SinPtr;
2279 0 : Entry.Ty = RetTy->getPointerTo();
2280 0 : Entry.IsSExt = false;
2281 0 : Entry.IsZExt = false;
2282 0 : Args.push_back(Entry);
2283 :
2284 : // Also pass the return address of the cos.
2285 0 : SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2286 0 : Entry.Node = CosPtr;
2287 0 : Entry.Ty = RetTy->getPointerTo();
2288 0 : Entry.IsSExt = false;
2289 0 : Entry.IsZExt = false;
2290 0 : Args.push_back(Entry);
2291 :
2292 0 : SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2293 0 : TLI.getPointerTy(DAG.getDataLayout()));
2294 :
2295 : SDLoc dl(Node);
2296 0 : TargetLowering::CallLoweringInfo CLI(DAG);
2297 0 : CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2298 0 : TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2299 0 : std::move(Args));
2300 :
2301 0 : std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2302 :
2303 0 : Results.push_back(
2304 0 : DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2305 0 : Results.push_back(
2306 0 : DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2307 0 : }
2308 :
2309 : /// This function is responsible for legalizing a
2310 : /// INT_TO_FP operation of the specified operand when the target requests that
2311 : /// we expand it. At this point, we know that the result and operand types are
2312 : /// legal for the target.
2313 3937 : SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2314 : EVT DestVT,
2315 : const SDLoc &dl) {
2316 3937 : EVT SrcVT = Op0.getValueType();
2317 3937 : EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2318 :
2319 : // TODO: Should any fast-math-flags be set for the created nodes?
2320 : LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2321 51 : if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2322 : LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2323 : "expansion\n");
2324 :
2325 : // Get the stack frame index of a 8 byte buffer.
2326 100 : SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2327 :
2328 : // word offset constant for Hi/Lo address computation
2329 50 : SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2330 50 : StackSlot.getValueType());
2331 : // set up Hi and Lo (into buffer) address based on endian
2332 50 : SDValue Hi = StackSlot;
2333 50 : SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2334 50 : StackSlot, WordOff);
2335 50 : if (DAG.getDataLayout().isLittleEndian())
2336 : std::swap(Hi, Lo);
2337 :
2338 : // if signed map to unsigned space
2339 50 : SDValue Op0Mapped;
2340 50 : if (isSigned) {
2341 : // constant used to invert sign bit (signed to unsigned mapping)
2342 8 : SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2343 8 : Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2344 : } else {
2345 46 : Op0Mapped = Op0;
2346 : }
2347 : // store the lo of the constructed double - based on integer input
2348 50 : SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2349 50 : MachinePointerInfo());
2350 : // initial hi portion of constructed double
2351 100 : SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2352 : // store the hi of the constructed double - biased exponent
2353 : SDValue Store2 =
2354 100 : DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2355 : // load the constructed double
2356 : SDValue Load =
2357 100 : DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2358 : // FP constant to bias correct the final result
2359 50 : SDValue Bias = DAG.getConstantFP(isSigned ?
2360 : BitsToDouble(0x4330000080000000ULL) :
2361 : BitsToDouble(0x4330000000000000ULL),
2362 96 : dl, MVT::f64);
2363 : // subtract the bias
2364 100 : SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2365 : // final result
2366 50 : SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2367 50 : return Result;
2368 : }
2369 : assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2370 : // Code below here assumes !isSigned without checking again.
2371 :
2372 : // Implementation of unsigned i64 to f64 following the algorithm in
2373 : // __floatundidf in compiler_rt. This implementation has the advantage
2374 : // of performing rounding correctly, both in the default rounding mode
2375 : // and in all alternate rounding modes.
2376 : // TODO: Generalize this for use with other types.
2377 : if (SrcVT == MVT::i64 && DestVT == MVT::f64) {
2378 : LLVM_DEBUG(dbgs() << "Converting unsigned i64 to f64\n");
2379 4 : SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
2380 4 : SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
2381 4 : BitsToDouble(UINT64_C(0x4530000000100000)), dl, DestVT);
2382 4 : SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
2383 4 : SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
2384 4 : SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
2385 :
2386 8 : SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Op0, LoMask);
2387 8 : SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, HiShift);
2388 8 : SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
2389 8 : SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
2390 8 : SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, DestVT, LoOr);
2391 8 : SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, DestVT, HiOr);
2392 8 : SDValue HiSub = DAG.getNode(ISD::FSUB, dl, DestVT, HiFlt, TwoP84PlusTwoP52);
2393 8 : return DAG.getNode(ISD::FADD, dl, DestVT, LoFlt, HiSub);
2394 : }
2395 :
2396 : // TODO: Generalize this for use with other types.
2397 : if (SrcVT == MVT::i64 && DestVT == MVT::f32) {
2398 : LLVM_DEBUG(dbgs() << "Converting unsigned i64 to f32\n");
2399 : // For unsigned conversions, convert them to signed conversions using the
2400 : // algorithm from the x86_64 __floatundidf in compiler_rt.
2401 3881 : if (!isSigned) {
2402 7762 : SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2403 :
2404 3881 : SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2405 7762 : SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2406 7762 : SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2407 7762 : SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2408 7762 : SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2409 :
2410 7762 : SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2411 7762 : SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2412 :
2413 : // TODO: This really should be implemented using a branch rather than a
2414 : // select. We happen to get lucky and machinesink does the right
2415 : // thing most of the time. This would be a good candidate for a
2416 : //pseudo-op, or, even better, for whole-function isel.
2417 3881 : SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2418 3881 : Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2419 7762 : return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2420 : }
2421 :
2422 : // Otherwise, implement the fully general conversion.
2423 :
2424 0 : SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2425 0 : DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
2426 0 : SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2427 0 : DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
2428 0 : SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2429 0 : DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
2430 0 : SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2431 : DAG.getConstant(UINT64_C(0), dl, MVT::i64),
2432 0 : ISD::SETNE);
2433 0 : SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2434 0 : SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2435 : DAG.getConstant(UINT64_C(0x0020000000000000), dl,
2436 : MVT::i64),
2437 0 : ISD::SETUGE);
2438 0 : SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2439 0 : EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2440 :
2441 0 : SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2442 0 : DAG.getConstant(32, dl, SHVT));
2443 0 : SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2444 0 : SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2445 : SDValue TwoP32 =
2446 0 : DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
2447 0 : MVT::f64);
2448 0 : SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2449 0 : SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2450 0 : SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2451 0 : SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2452 0 : return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2453 0 : DAG.getIntPtrConstant(0, dl));
2454 : }
2455 :
2456 4 : SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2457 :
2458 2 : SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2459 2 : DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2460 2 : SDValue Zero = DAG.getIntPtrConstant(0, dl),
2461 2 : Four = DAG.getIntPtrConstant(4, dl);
2462 2 : SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2463 4 : SignSet, Four, Zero);
2464 :
2465 : // If the sign bit of the integer is set, the large number will be treated
2466 : // as a negative number. To counteract this, the dynamic code adds an
2467 : // offset depending on the data type.
2468 : uint64_t FF;
2469 : switch (SrcVT.getSimpleVT().SimpleTy) {
2470 0 : default: llvm_unreachable("Unsupported integer type!");
2471 : case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2472 : case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2473 : case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2474 : case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2475 : }
2476 2 : if (DAG.getDataLayout().isLittleEndian())
2477 0 : FF <<= 32;
2478 2 : Constant *FudgeFactor = ConstantInt::get(
2479 2 : Type::getInt64Ty(*DAG.getContext()), FF);
2480 :
2481 : SDValue CPIdx =
2482 4 : DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2483 2 : unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2484 4 : CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2485 2 : Alignment = std::min(Alignment, 4u);
2486 2 : SDValue FudgeInReg;
2487 : if (DestVT == MVT::f32)
2488 1 : FudgeInReg = DAG.getLoad(
2489 1 : MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2490 : MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2491 1 : Alignment);
2492 : else {
2493 1 : SDValue Load = DAG.getExtLoad(
2494 1 : ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2495 : MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2496 1 : Alignment);
2497 1 : HandleSDNode Handle(Load);
2498 1 : LegalizeOp(Load.getNode());
2499 1 : FudgeInReg = Handle.getValue();
2500 : }
2501 :
2502 4 : return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2503 : }
2504 :
2505 : /// This function is responsible for legalizing a
2506 : /// *INT_TO_FP operation of the specified operand when the target requests that
2507 : /// we promote it. At this point, we know that the result and operand types are
2508 : /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2509 : /// operation that takes a larger input.
2510 1129 : SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2511 : bool isSigned,
2512 : const SDLoc &dl) {
2513 : // First step, figure out the appropriate *INT_TO_FP operation to use.
2514 2258 : EVT NewInTy = LegalOp.getValueType();
2515 :
2516 : unsigned OpToUse = 0;
2517 :
2518 : // Scan for the appropriate larger type to use.
2519 : while (true) {
2520 1135 : NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2521 : assert(NewInTy.isInteger() && "Ran out of possibilities!");
2522 :
2523 : // If the target supports SINT_TO_FP of this type, use it.
2524 1135 : if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2525 : OpToUse = ISD::SINT_TO_FP;
2526 : break;
2527 : }
2528 6 : if (isSigned) continue;
2529 :
2530 : // If the target supports UINT_TO_FP of this type, use it.
2531 : if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2532 : OpToUse = ISD::UINT_TO_FP;
2533 : break;
2534 : }
2535 :
2536 : // Otherwise, try a larger type.
2537 : }
2538 :
2539 : // Okay, we found the operation and type to use. Zero extend our input to the
2540 : // desired type then run the operation on it.
2541 1129 : return DAG.getNode(OpToUse, dl, DestVT,
2542 : DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2543 2225 : dl, NewInTy, LegalOp));
2544 : }
2545 :
2546 : /// This function is responsible for legalizing a
2547 : /// FP_TO_*INT operation of the specified operand when the target requests that
2548 : /// we promote it. At this point, we know that the result and operand types are
2549 : /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2550 : /// operation that returns a larger result.
2551 255 : SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2552 : bool isSigned,
2553 : const SDLoc &dl) {
2554 : // First step, figure out the appropriate FP_TO*INT operation to use.
2555 255 : EVT NewOutTy = DestVT;
2556 :
2557 : unsigned OpToUse = 0;
2558 :
2559 : // Scan for the appropriate larger type to use.
2560 : while (true) {
2561 299 : NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2562 : assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2563 :
2564 : // A larger signed type can hold all unsigned values of the requested type,
2565 : // so using FP_TO_SINT is valid
2566 299 : if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2567 : OpToUse = ISD::FP_TO_SINT;
2568 : break;
2569 : }
2570 :
2571 : // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2572 44 : if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2573 : OpToUse = ISD::FP_TO_UINT;
2574 : break;
2575 : }
2576 :
2577 : // Otherwise, try a larger type.
2578 : }
2579 :
2580 : // Okay, we found the operation and type to use.
2581 510 : SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2582 :
2583 : // Truncate the result of the extended FP_TO_*INT operation to the desired
2584 : // size.
2585 510 : return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2586 : }
2587 :
2588 : /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2589 0 : SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2590 0 : EVT VT = Op.getValueType();
2591 0 : EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2592 : unsigned Sz = VT.getScalarSizeInBits();
2593 :
2594 0 : SDValue Tmp, Tmp2, Tmp3;
2595 :
2596 : // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2597 : // and finally the i1 pairs.
2598 : // TODO: We can easily support i4/i2 legal types if any target ever does.
2599 0 : if (Sz >= 8 && isPowerOf2_32(Sz)) {
2600 : // Create the masks - repeating the pattern every byte.
2601 : APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2602 : APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2603 0 : for (unsigned J = 0; J != Sz; J += 8) {
2604 0 : MaskHi4 = MaskHi4 | (0xF0ull << J);
2605 0 : MaskLo4 = MaskLo4 | (0x0Full << J);
2606 0 : MaskHi2 = MaskHi2 | (0xCCull << J);
2607 0 : MaskLo2 = MaskLo2 | (0x33ull << J);
2608 0 : MaskHi1 = MaskHi1 | (0xAAull << J);
2609 0 : MaskLo1 = MaskLo1 | (0x55ull << J);
2610 : }
2611 :
2612 : // BSWAP if the type is wider than a single byte.
2613 0 : Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2614 :
2615 : // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2616 0 : Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2617 0 : Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2618 0 : Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2619 0 : Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2620 0 : Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2621 :
2622 : // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2623 0 : Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2624 0 : Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2625 0 : Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2626 0 : Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2627 0 : Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2628 :
2629 : // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2630 0 : Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2631 0 : Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2632 0 : Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2633 0 : Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2634 0 : Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2635 0 : return Tmp;
2636 : }
2637 :
2638 0 : Tmp = DAG.getConstant(0, dl, VT);
2639 0 : for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2640 0 : if (I < J)
2641 0 : Tmp2 =
2642 0 : DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2643 : else
2644 0 : Tmp2 =
2645 0 : DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2646 :
2647 : APInt Shift(Sz, 1);
2648 0 : Shift <<= J;
2649 0 : Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2650 0 : Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2651 : }
2652 :
2653 0 : return Tmp;
2654 : }
2655 :
2656 : /// Open code the operations for BSWAP of the specified operation.
2657 0 : SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2658 0 : EVT VT = Op.getValueType();
2659 0 : EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2660 0 : SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2661 0 : switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2662 0 : default: llvm_unreachable("Unhandled Expand type in BSWAP!");
2663 0 : case MVT::i16:
2664 0 : Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2665 0 : Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2666 0 : return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2667 0 : case MVT::i32:
2668 0 : Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2669 0 : Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2670 0 : Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2671 0 : Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2672 0 : Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2673 0 : DAG.getConstant(0xFF0000, dl, VT));
2674 0 : Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2675 0 : Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2676 0 : Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2677 0 : return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2678 0 : case MVT::i64:
2679 0 : Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2680 0 : Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2681 0 : Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2682 0 : Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2683 0 : Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2684 0 : Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2685 0 : Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2686 0 : Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2687 0 : Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2688 0 : DAG.getConstant(255ULL<<48, dl, VT));
2689 0 : Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2690 0 : DAG.getConstant(255ULL<<40, dl, VT));
2691 0 : Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2692 0 : DAG.getConstant(255ULL<<32, dl, VT));
2693 0 : Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2694 0 : DAG.getConstant(255ULL<<24, dl, VT));
2695 0 : Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2696 0 : DAG.getConstant(255ULL<<16, dl, VT));
2697 0 : Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2698 0 : DAG.getConstant(255ULL<<8 , dl, VT));
2699 0 : Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2700 0 : Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2701 0 : Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2702 0 : Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2703 0 : Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2704 0 : Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2705 0 : return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2706 : }
2707 : }
2708 :
2709 : /// Expand the specified bitcount instruction into operations.
2710 774 : SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2711 : const SDLoc &dl) {
2712 774 : EVT VT = Op.getValueType();
2713 774 : EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2714 : unsigned Len = VT.getScalarSizeInBits();
2715 :
2716 774 : switch (Opc) {
2717 0 : default: llvm_unreachable("Cannot expand this yet!");
2718 105 : case ISD::CTPOP: {
2719 : assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
2720 : "CTPOP not implemented for this type.");
2721 :
2722 : // This is the "best" algorithm from
2723 : // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2724 :
2725 210 : SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2726 105 : dl, VT);
2727 210 : SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2728 105 : dl, VT);
2729 210 : SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2730 105 : dl, VT);
2731 210 : SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2732 105 : dl, VT);
2733 :
2734 : // v = v - ((v >> 1) & 0x55555555...)
2735 105 : Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2736 : DAG.getNode(ISD::AND, dl, VT,
2737 : DAG.getNode(ISD::SRL, dl, VT, Op,
2738 : DAG.getConstant(1, dl, ShVT)),
2739 105 : Mask55));
2740 : // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2741 105 : Op = DAG.getNode(ISD::ADD, dl, VT,
2742 105 : DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2743 : DAG.getNode(ISD::AND, dl, VT,
2744 : DAG.getNode(ISD::SRL, dl, VT, Op,
2745 : DAG.getConstant(2, dl, ShVT)),
2746 105 : Mask33));
2747 : // v = (v + (v >> 4)) & 0x0F0F0F0F...
2748 105 : Op = DAG.getNode(ISD::AND, dl, VT,
2749 : DAG.getNode(ISD::ADD, dl, VT, Op,
2750 : DAG.getNode(ISD::SRL, dl, VT, Op,
2751 : DAG.getConstant(4, dl, ShVT))),
2752 105 : Mask0F);
2753 : // v = (v * 0x01010101...) >> (Len - 8)
2754 105 : if (Len > 8)
2755 99 : Op = DAG.getNode(ISD::SRL, dl, VT,
2756 99 : DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2757 99 : DAG.getConstant(Len - 8, dl, ShVT));
2758 :
2759 105 : return Op;
2760 : }
2761 127 : case ISD::CTLZ_ZERO_UNDEF:
2762 : // This trivially expands to CTLZ.
2763 254 : return DAG.getNode(ISD::CTLZ, dl, VT, Op);
2764 84 : case ISD::CTLZ: {
2765 84 : if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
2766 40 : EVT SetCCVT = getSetCCResultType(VT);
2767 80 : SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2768 40 : SDValue Zero = DAG.getConstant(0, dl, VT);
2769 40 : SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2770 40 : return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2771 40 : DAG.getConstant(Len, dl, VT), CTLZ);
2772 : }
2773 :
2774 : // for now, we do this:
2775 : // x = x | (x >> 1);
2776 : // x = x | (x >> 2);
2777 : // ...
2778 : // x = x | (x >>16);
2779 : // x = x | (x >>32); // for 64-bit input
2780 : // return popcount(~x);
2781 : //
2782 : // Ref: "Hacker's Delight" by Henry Warren
2783 274 : for (unsigned i = 0; (1U << i) <= (Len / 2); ++i) {
2784 230 : SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2785 230 : Op = DAG.getNode(ISD::OR, dl, VT, Op,
2786 230 : DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2787 : }
2788 44 : Op = DAG.getNOT(dl, Op, VT);
2789 88 : return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2790 : }
2791 193 : case ISD::CTTZ_ZERO_UNDEF:
2792 : // This trivially expands to CTTZ.
2793 386 : return DAG.getNode(ISD::CTTZ, dl, VT, Op);
2794 265 : case ISD::CTTZ: {
2795 265 : if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
2796 17 : EVT SetCCVT = getSetCCResultType(VT);
2797 34 : SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
2798 17 : SDValue Zero = DAG.getConstant(0, dl, VT);
2799 17 : SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2800 17 : return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2801 17 : DAG.getConstant(Len, dl, VT), CTTZ);
2802 : }
2803 :
2804 : // for now, we use: { return popcount(~x & (x - 1)); }
2805 : // unless the target has ctlz but not ctpop, in which case we use:
2806 : // { return 32 - nlz(~x & (x-1)); }
2807 : // Ref: "Hacker's Delight" by Henry Warren
2808 248 : SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2809 248 : DAG.getNOT(dl, Op, VT),
2810 : DAG.getNode(ISD::SUB, dl, VT, Op,
2811 248 : DAG.getConstant(1, dl, VT)));
2812 : // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2813 248 : if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
2814 : TLI.isOperationLegal(ISD::CTLZ, VT))
2815 29 : return DAG.getNode(ISD::SUB, dl, VT,
2816 29 : DAG.getConstant(Len, dl, VT),
2817 29 : DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2818 438 : return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2819 : }
2820 : }
2821 : }
2822 :
2823 345107 : bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2824 : LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2825 : SmallVector<SDValue, 8> Results;
2826 : SDLoc dl(Node);
2827 345107 : SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2828 : bool NeedInvert;
2829 690214 : switch (Node->getOpcode()) {
2830 774 : case ISD::CTPOP:
2831 : case ISD::CTLZ:
2832 : case ISD::CTLZ_ZERO_UNDEF:
2833 : case ISD::CTTZ:
2834 : case ISD::CTTZ_ZERO_UNDEF:
2835 1548 : Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2836 774 : Results.push_back(Tmp1);
2837 774 : break;
2838 118 : case ISD::BITREVERSE:
2839 236 : Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2840 118 : break;
2841 133 : case ISD::BSWAP:
2842 266 : Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2843 133 : break;
2844 2 : case ISD::FRAMEADDR:
2845 : case ISD::RETURNADDR:
2846 : case ISD::FRAME_TO_ARGS_OFFSET:
2847 4 : Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2848 2 : break;
2849 6 : case ISD::EH_DWARF_CFA: {
2850 6 : SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2851 12 : TLI.getPointerTy(DAG.getDataLayout()));
2852 6 : SDValue Offset = DAG.getNode(ISD::ADD, dl,
2853 : CfaArg.getValueType(),
2854 : DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2855 : CfaArg.getValueType()),
2856 6 : CfaArg);
2857 6 : SDValue FA = DAG.getNode(
2858 6 : ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2859 18 : DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2860 6 : Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2861 18 : FA, Offset));
2862 : break;
2863 : }
2864 1 : case ISD::FLT_ROUNDS_:
2865 2 : Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2866 1 : break;
2867 322 : case ISD::EH_RETURN:
2868 : case ISD::EH_LABEL:
2869 : case ISD::PREFETCH:
2870 : case ISD::VAEND:
2871 : case ISD::EH_SJLJ_LONGJMP:
2872 : // If the target didn't expand these, there's nothing to do, so just
2873 : // preserve the chain and be done.
2874 644 : Results.push_back(Node->getOperand(0));
2875 322 : break;
2876 3 : case ISD::READCYCLECOUNTER:
2877 : // If the target didn't expand this, just return 'zero' and preserve the
2878 : // chain.
2879 6 : Results.append(Node->getNumValues() - 1,
2880 6 : DAG.getConstant(0, dl, Node->getValueType(0)));
2881 6 : Results.push_back(Node->getOperand(0));
2882 3 : break;
2883 0 : case ISD::EH_SJLJ_SETJMP:
2884 : // If the target didn't expand this, just return 'zero' and preserve the
2885 : // chain.
2886 0 : Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2887 0 : Results.push_back(Node->getOperand(0));
2888 0 : break;
2889 6 : case ISD::ATOMIC_LOAD: {
2890 : // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2891 12 : SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2892 18 : SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2893 6 : SDValue Swap = DAG.getAtomicCmpSwap(
2894 : ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2895 6 : Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2896 12 : cast<AtomicSDNode>(Node)->getMemOperand());
2897 6 : Results.push_back(Swap.getValue(0));
2898 6 : Results.push_back(Swap.getValue(1));
2899 : break;
2900 : }
2901 14 : case ISD::ATOMIC_STORE: {
2902 : // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2903 14 : SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2904 : cast<AtomicSDNode>(Node)->getMemoryVT(),
2905 : Node->getOperand(0),
2906 14 : Node->getOperand(1), Node->getOperand(2),
2907 28 : cast<AtomicSDNode>(Node)->getMemOperand());
2908 14 : Results.push_back(Swap.getValue(1));
2909 : break;
2910 : }
2911 5882 : case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2912 : // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2913 : // splits out the success value as a comparison. Expanding the resulting
2914 : // ATOMIC_CMP_SWAP will produce a libcall.
2915 17646 : SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2916 5882 : SDValue Res = DAG.getAtomicCmpSwap(
2917 : ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2918 : Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2919 11764 : Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2920 :
2921 5882 : SDValue ExtRes = Res;
2922 5882 : SDValue LHS = Res;
2923 5882 : SDValue RHS = Node->getOperand(1);
2924 :
2925 5882 : EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2926 5882 : EVT OuterType = Node->getValueType(0);
2927 5882 : switch (TLI.getExtendForAtomicOps()) {
2928 89 : case ISD::SIGN_EXTEND:
2929 89 : LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2930 89 : DAG.getValueType(AtomicType));
2931 89 : RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2932 89 : Node->getOperand(2), DAG.getValueType(AtomicType));
2933 89 : ExtRes = LHS;
2934 89 : break;
2935 5742 : case ISD::ZERO_EXTEND:
2936 5742 : LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2937 5742 : DAG.getValueType(AtomicType));
2938 11484 : RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2939 5742 : ExtRes = LHS;
2940 5742 : break;
2941 51 : case ISD::ANY_EXTEND:
2942 51 : LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2943 102 : RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2944 51 : break;
2945 0 : default:
2946 0 : llvm_unreachable("Invalid atomic op extension");
2947 : }
2948 :
2949 : SDValue Success =
2950 11764 : DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2951 :
2952 5882 : Results.push_back(ExtRes.getValue(0));
2953 5882 : Results.push_back(Success);
2954 5882 : Results.push_back(Res.getValue(1));
2955 : break;
2956 : }
2957 158 : case ISD::DYNAMIC_STACKALLOC:
2958 158 : ExpandDYNAMIC_STACKALLOC(Node, Results);
2959 158 : break;
2960 : case ISD::MERGE_VALUES:
2961 925766 : for (unsigned i = 0; i < Node->getNumValues(); i++)
2962 1157293 : Results.push_back(Node->getOperand(i));
2963 : break;
2964 236 : case ISD::UNDEF: {
2965 236 : EVT VT = Node->getValueType(0);
2966 236 : if (VT.isInteger())
2967 154 : Results.push_back(DAG.getConstant(0, dl, VT));
2968 : else {
2969 : assert(VT.isFloatingPoint() && "Unknown value type!");
2970 82 : Results.push_back(DAG.getConstantFP(0, dl, VT));
2971 : }
2972 : break;
2973 : }
2974 395 : case ISD::FP_ROUND:
2975 : case ISD::BITCAST:
2976 395 : Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2977 790 : Node->getValueType(0), dl);
2978 395 : Results.push_back(Tmp1);
2979 395 : break;
2980 0 : case ISD::FP_EXTEND:
2981 0 : Tmp1 = EmitStackConvert(Node->getOperand(0),
2982 0 : Node->getOperand(0).getValueType(),
2983 0 : Node->getValueType(0), dl);
2984 0 : Results.push_back(Tmp1);
2985 0 : break;
2986 1302 : case ISD::SIGN_EXTEND_INREG: {
2987 1302 : EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2988 1302 : EVT VT = Node->getValueType(0);
2989 :
2990 : // An in-register sign-extend of a boolean is a negation:
2991 : // 'true' (1) sign-extended is -1.
2992 : // 'false' (0) sign-extended is 0.
2993 : // However, we must mask the high bits of the source operand because the
2994 : // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2995 :
2996 : // TODO: Do this for vectors too?
2997 1302 : if (ExtraVT.getSizeInBits() == 1) {
2998 317 : SDValue One = DAG.getConstant(1, dl, VT);
2999 951 : SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3000 317 : SDValue Zero = DAG.getConstant(0, dl, VT);
3001 634 : SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3002 317 : Results.push_back(Neg);
3003 : break;
3004 : }
3005 :
3006 : // NOTE: we could fall back on load/store here too for targets without
3007 : // SRA. However, it is doubtful that any exist.
3008 985 : EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3009 : unsigned BitsDiff = VT.getScalarSizeInBits() -
3010 985 : ExtraVT.getScalarSizeInBits();
3011 985 : SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3012 985 : Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3013 2955 : Node->getOperand(0), ShiftCst);
3014 2955 : Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3015 985 : Results.push_back(Tmp1);
3016 985 : break;
3017 : }
3018 0 : case ISD::FP_ROUND_INREG: {
3019 : // The only way we can lower this is to turn it into a TRUNCSTORE,
3020 : // EXTLOAD pair, targeting a temporary location (a stack slot).
3021 :
3022 : // NOTE: there is a choice here between constantly creating new stack
3023 : // slots and always reusing the same one. We currently always create
3024 : // new ones, as reuse may inhibit scheduling.
3025 0 : EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3026 0 : Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3027 0 : Node->getValueType(0), dl);
3028 0 : Results.push_back(Tmp1);
3029 : break;
3030 : }
3031 3937 : case ISD::SINT_TO_FP:
3032 : case ISD::UINT_TO_FP:
3033 3937 : Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3034 11811 : Node->getOperand(0), Node->getValueType(0), dl);
3035 3937 : Results.push_back(Tmp1);
3036 3937 : break;
3037 61 : case ISD::FP_TO_SINT:
3038 61 : if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3039 61 : Results.push_back(Tmp1);
3040 : break;
3041 2043 : case ISD::FP_TO_UINT: {
3042 2043 : SDValue True, False;
3043 2043 : EVT VT = Node->getOperand(0).getValueType();
3044 2043 : EVT NVT = Node->getValueType(0);
3045 : APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3046 2043 : APInt::getNullValue(VT.getSizeInBits()));
3047 2043 : APInt x = APInt::getSignMask(NVT.getSizeInBits());
3048 2043 : (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3049 2043 : Tmp1 = DAG.getConstantFP(apf, dl, VT);
3050 4086 : Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3051 2043 : Node->getOperand(0),
3052 2043 : Tmp1, ISD::SETLT);
3053 6129 : True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3054 : // TODO: Should any fast-math-flags be set for the FSUB?
3055 2043 : False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3056 : DAG.getNode(ISD::FSUB, dl, VT,
3057 4086 : Node->getOperand(0), Tmp1));
3058 2043 : False = DAG.getNode(ISD::XOR, dl, NVT, False,
3059 2043 : DAG.getConstant(x, dl, NVT));
3060 2043 : Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3061 2043 : Results.push_back(Tmp1);
3062 : break;
3063 : }
3064 30 : case ISD::VAARG:
3065 30 : Results.push_back(DAG.expandVAArg(Node));
3066 30 : Results.push_back(Results[0].getValue(1));
3067 30 : break;
3068 3 : case ISD::VACOPY:
3069 3 : Results.push_back(DAG.expandVACopy(Node));
3070 3 : break;
3071 1826 : case ISD::EXTRACT_VECTOR_ELT:
3072 5478 : if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3073 : // This must be an access of the only element. Return it.
3074 69 : Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3075 138 : Node->getOperand(0));
3076 : else
3077 1757 : Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3078 1826 : Results.push_back(Tmp1);
3079 1826 : break;
3080 : case ISD::EXTRACT_SUBVECTOR:
3081 0 : Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3082 0 : break;
3083 : case ISD::INSERT_SUBVECTOR:
3084 0 : Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3085 0 : break;
3086 0 : case ISD::CONCAT_VECTORS:
3087 0 : Results.push_back(ExpandVectorBuildThroughStack(Node));
3088 0 : break;
3089 18 : case ISD::SCALAR_TO_VECTOR:
3090 18 : Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3091 18 : break;
3092 1411 : case ISD::INSERT_VECTOR_ELT:
3093 1411 : Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3094 : Node->getOperand(1),
3095 2822 : Node->getOperand(2), dl));
3096 1411 : break;
3097 : case ISD::VECTOR_SHUFFLE: {
3098 : SmallVector<int, 32> NewMask;
3099 304 : ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3100 :
3101 304 : EVT VT = Node->getValueType(0);
3102 304 : EVT EltVT = VT.getVectorElementType();
3103 304 : SDValue Op0 = Node->getOperand(0);
3104 304 : SDValue Op1 = Node->getOperand(1);
3105 304 : if (!TLI.isTypeLegal(EltVT)) {
3106 7 : EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3107 :
3108 : // BUILD_VECTOR operands are allowed to be wider than the element type.
3109 : // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3110 : // it.
3111 7 : if (NewEltVT.bitsLT(EltVT)) {
3112 : // Convert shuffle node.
3113 : // If original node was v4i64 and the new EltVT is i32,
3114 : // cast operands to v8i32 and re-build the mask.
3115 :
3116 : // Calculate new VT, the size of the new VT should be equal to original.
3117 : EVT NewVT =
3118 0 : EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3119 0 : VT.getSizeInBits() / NewEltVT.getSizeInBits());
3120 : assert(NewVT.bitsEq(VT));
3121 :
3122 : // cast operands to new VT
3123 0 : Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3124 0 : Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3125 :
3126 : // Convert the shuffle mask
3127 : unsigned int factor =
3128 0 : NewVT.getVectorNumElements()/VT.getVectorNumElements();
3129 :
3130 : // EltVT gets smaller
3131 : assert(factor > 0);
3132 :
3133 0 : for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3134 0 : if (Mask[i] < 0) {
3135 0 : for (unsigned fi = 0; fi < factor; ++fi)
3136 0 : NewMask.push_back(Mask[i]);
3137 : }
3138 : else {
3139 0 : for (unsigned fi = 0; fi < factor; ++fi)
3140 0 : NewMask.push_back(Mask[i]*factor+fi);
3141 : }
3142 : }
3143 : Mask = NewMask;
3144 0 : VT = NewVT;
3145 : }
3146 7 : EltVT = NewEltVT;
3147 : }
3148 : unsigned NumElems = VT.getVectorNumElements();
3149 : SmallVector<SDValue, 16> Ops;
3150 1190 : for (unsigned i = 0; i != NumElems; ++i) {
3151 1772 : if (Mask[i] < 0) {
3152 40 : Ops.push_back(DAG.getUNDEF(EltVT));
3153 40 : continue;
3154 : }
3155 846 : unsigned Idx = Mask[i];
3156 846 : if (Idx < NumElems)
3157 978 : Ops.push_back(DAG.getNode(
3158 : ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3159 978 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3160 : else
3161 714 : Ops.push_back(DAG.getNode(
3162 : ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3163 357 : DAG.getConstant(Idx - NumElems, dl,
3164 714 : TLI.getVectorIdxTy(DAG.getDataLayout()))));
3165 : }
3166 :
3167 608 : Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3168 : // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3169 912 : Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3170 304 : Results.push_back(Tmp1);
3171 : break;
3172 : }
3173 359 : case ISD::EXTRACT_ELEMENT: {
3174 718 : EVT OpTy = Node->getOperand(0).getValueType();
3175 718 : if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3176 : // 1 -> Hi
3177 322 : Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3178 161 : DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3179 161 : TLI.getShiftAmountTy(
3180 161 : Node->getOperand(0).getValueType(),
3181 161 : DAG.getDataLayout())));
3182 483 : Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3183 : } else {
3184 : // 0 -> Lo
3185 198 : Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3186 396 : Node->getOperand(0));
3187 : }
3188 359 : Results.push_back(Tmp1);
3189 : break;
3190 : }
3191 163 : case ISD::STACKSAVE:
3192 : // Expand to CopyFromReg if the target set
3193 : // StackPointerRegisterToSaveRestore.
3194 163 : if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3195 163 : Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3196 326 : Node->getValueType(0)));
3197 163 : Results.push_back(Results[0].getValue(1));
3198 : } else {
3199 0 : Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3200 0 : Results.push_back(Node->getOperand(0));
3201 : }
3202 : break;
3203 72 : case ISD::STACKRESTORE:
3204 : // Expand to CopyToReg if the target set
3205 : // StackPointerRegisterToSaveRestore.
3206 72 : if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3207 72 : Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3208 144 : Node->getOperand(1)));
3209 : } else {
3210 0 : Results.push_back(Node->getOperand(0));
3211 : }
3212 : break;
3213 0 : case ISD::GET_DYNAMIC_AREA_OFFSET:
3214 0 : Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3215 0 : Results.push_back(Results[0].getValue(0));
3216 0 : break;
3217 200 : case ISD::FCOPYSIGN:
3218 200 : Results.push_back(ExpandFCOPYSIGN(Node));
3219 200 : break;
3220 21 : case ISD::FNEG:
3221 : // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3222 42 : Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3223 : // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3224 21 : Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3225 63 : Node->getOperand(0));
3226 21 : Results.push_back(Tmp1);
3227 21 : break;
3228 2 : case ISD::FABS:
3229 2 : Results.push_back(ExpandFABS(Node));
3230 2 : break;
3231 0 : case ISD::SMIN:
3232 : case ISD::SMAX:
3233 : case ISD::UMIN:
3234 : case ISD::UMAX: {
3235 : // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3236 : ISD::CondCode Pred;
3237 : switch (Node->getOpcode()) {
3238 0 : default: llvm_unreachable("How did we get here?");
3239 : case ISD::SMAX: Pred = ISD::SETGT; break;
3240 : case ISD::SMIN: Pred = ISD::SETLT; break;
3241 : case ISD::UMAX: Pred = ISD::SETUGT; break;
3242 : case ISD::UMIN: Pred = ISD::SETULT; break;
3243 : }
3244 0 : Tmp1 = Node->getOperand(0);
3245 0 : Tmp2 = Node->getOperand(1);
3246 0 : Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3247 0 : Results.push_back(Tmp1);
3248 0 : break;
3249 : }
3250 :
3251 363 : case ISD::FSIN:
3252 : case ISD::FCOS: {
3253 363 : EVT VT = Node->getValueType(0);
3254 : // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3255 : // fcos which share the same operand and both are used.
3256 702 : if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3257 : isSinCosLibcallAvailable(Node, TLI))
3258 159 : && useSinCos(Node)) {
3259 88 : SDVTList VTs = DAG.getVTList(VT, VT);
3260 176 : Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3261 88 : if (Node->getOpcode() == ISD::FCOS)
3262 44 : Tmp1 = Tmp1.getValue(1);
3263 88 : Results.push_back(Tmp1);
3264 : }
3265 : break;
3266 : }
3267 : case ISD::FMAD:
3268 : llvm_unreachable("Illegal fmad should never be formed");
3269 :
3270 : case ISD::FP16_TO_FP:
3271 357 : if (Node->getValueType(0) != MVT::f32) {
3272 : // We can extend to types bigger than f32 in two steps without changing
3273 : // the result. Since "f16 -> f32" is much more commonly available, give
3274 : // CodeGen the option of emitting that before resorting to a libcall.
3275 : SDValue Res =
3276 0 : DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3277 0 : Results.push_back(
3278 0 : DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3279 : }
3280 : break;
3281 483 : case ISD::FP_TO_FP16:
3282 : LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3283 483 : if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3284 15 : SDValue Op = Node->getOperand(0);
3285 : MVT SVT = Op.getSimpleValueType();
3286 15 : if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3287 15 : TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3288 : // Under fastmath, we can expand this node into a fround followed by
3289 : // a float-half conversion.
3290 13 : SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3291 13 : DAG.getIntPtrConstant(0, dl));
3292 13 : Results.push_back(
3293 52 : DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3294 : }
3295 : }
3296 : break;
3297 : case ISD::ConstantFP: {
3298 : ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3299 : // Check to see if this FP immediate is already legal.
3300 : // If this is a legal constant, turn it into a TargetConstantFP node.
3301 34944 : if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3302 7285 : Results.push_back(ExpandConstantFP(CFP, true));
3303 : break;
3304 : }
3305 : case ISD::Constant: {
3306 : ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3307 0 : Results.push_back(ExpandConstant(CP));
3308 0 : break;
3309 : }
3310 249 : case ISD::FSUB: {
3311 249 : EVT VT = Node->getValueType(0);
3312 249 : if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3313 : TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3314 174 : const SDNodeFlags Flags = Node->getFlags();
3315 522 : Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3316 348 : Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3317 174 : Results.push_back(Tmp1);
3318 : }
3319 : break;
3320 : }
3321 0 : case ISD::SUB: {
3322 0 : EVT VT = Node->getValueType(0);
3323 : assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3324 : TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3325 : "Don't know how to expand this subtraction!");
3326 0 : Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3327 0 : DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3328 0 : VT));
3329 0 : Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3330 0 : Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3331 : break;
3332 : }
3333 1064 : case ISD::UREM:
3334 : case ISD::SREM: {
3335 1064 : EVT VT = Node->getValueType(0);
3336 : bool isSigned = Node->getOpcode() == ISD::SREM;
3337 1064 : unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3338 1064 : unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3339 1064 : Tmp2 = Node->getOperand(0);
3340 1064 : Tmp3 = Node->getOperand(1);
3341 1064 : if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3342 866 : SDVTList VTs = DAG.getVTList(VT, VT);
3343 866 : Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3344 866 : Results.push_back(Tmp1);
3345 198 : } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3346 : // X % Y -> X-X/Y*Y
3347 324 : Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3348 324 : Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3349 324 : Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3350 162 : Results.push_back(Tmp1);
3351 : }
3352 : break;
3353 : }
3354 882 : case ISD::UDIV:
3355 : case ISD::SDIV: {
3356 : bool isSigned = Node->getOpcode() == ISD::SDIV;
3357 882 : unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3358 882 : EVT VT = Node->getValueType(0);
3359 882 : if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3360 873 : SDVTList VTs = DAG.getVTList(VT, VT);
3361 1746 : Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3362 1746 : Node->getOperand(1));
3363 873 : Results.push_back(Tmp1);
3364 : }
3365 : break;
3366 : }
3367 514 : case ISD::MULHU:
3368 : case ISD::MULHS: {
3369 : unsigned ExpandOpcode =
3370 514 : Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3371 514 : EVT VT = Node->getValueType(0);
3372 514 : SDVTList VTs = DAG.getVTList(VT, VT);
3373 :
3374 1028 : Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3375 1028 : Node->getOperand(1));
3376 514 : Results.push_back(Tmp1.getValue(1));
3377 : break;
3378 : }
3379 570 : case ISD::UMUL_LOHI:
3380 : case ISD::SMUL_LOHI: {
3381 570 : SDValue LHS = Node->getOperand(0);
3382 570 : SDValue RHS = Node->getOperand(1);
3383 : MVT VT = LHS.getSimpleValueType();
3384 : unsigned MULHOpcode =
3385 570 : Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3386 :
3387 1140 : if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3388 912 : Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3389 912 : Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3390 456 : break;
3391 : }
3392 :
3393 : SmallVector<SDValue, 4> Halves;
3394 114 : EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3395 : assert(TLI.isTypeLegal(HalfType));
3396 228 : if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3397 : HalfType, DAG,
3398 114 : TargetLowering::MulExpansionKind::Always)) {
3399 342 : for (unsigned i = 0; i < 2; ++i) {
3400 684 : SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3401 684 : SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3402 228 : SDValue Shift = DAG.getConstant(
3403 : HalfType.getScalarSizeInBits(), dl,
3404 456 : TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3405 456 : Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3406 456 : Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3407 : }
3408 : break;
3409 : }
3410 : break;
3411 : }
3412 608 : case ISD::MUL: {
3413 608 : EVT VT = Node->getValueType(0);
3414 608 : SDVTList VTs = DAG.getVTList(VT, VT);
3415 : // See if multiply or divide can be lowered using two-result operations.
3416 : // We just need the low half of the multiply; try both the signed
3417 : // and unsigned forms. If the target supports both SMUL_LOHI and
3418 : // UMUL_LOHI, form a preference by checking which forms of plain
3419 : // MULH it supports.
3420 608 : bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3421 : bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3422 : bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3423 : bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3424 : unsigned OpToUse = 0;
3425 608 : if (HasSMUL_LOHI && !HasMULHS) {
3426 : OpToUse = ISD::SMUL_LOHI;
3427 598 : } else if (HasUMUL_LOHI && !HasMULHU) {
3428 : OpToUse = ISD::UMUL_LOHI;
3429 598 : } else if (HasSMUL_LOHI) {
3430 : OpToUse = ISD::SMUL_LOHI;
3431 598 : } else if (HasUMUL_LOHI) {
3432 : OpToUse = ISD::UMUL_LOHI;
3433 : }
3434 : if (OpToUse) {
3435 10 : Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3436 20 : Node->getOperand(1)));
3437 10 : break;
3438 : }
3439 :
3440 598 : SDValue Lo, Hi;
3441 598 : EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3442 598 : if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3443 : TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3444 : TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3445 598 : TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3446 6 : TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3447 598 : TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3448 1184 : Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3449 1184 : Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3450 : SDValue Shift =
3451 1184 : DAG.getConstant(HalfType.getSizeInBits(), dl,
3452 592 : TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3453 1184 : Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3454 1184 : Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3455 : }
3456 : break;
3457 : }
3458 13 : case ISD::SADDSAT: {
3459 13 : Results.push_back(TLI.getExpandedSignedSaturationAddition(Node, DAG));
3460 13 : break;
3461 : }
3462 35 : case ISD::SADDO:
3463 : case ISD::SSUBO: {
3464 35 : SDValue LHS = Node->getOperand(0);
3465 35 : SDValue RHS = Node->getOperand(1);
3466 35 : SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3467 : ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3468 53 : LHS, RHS);
3469 35 : Results.push_back(Sum);
3470 70 : EVT ResultType = Node->getValueType(1);
3471 35 : EVT OType = getSetCCResultType(Node->getValueType(0));
3472 :
3473 70 : SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3474 :
3475 : // LHSSign -> LHS >= 0
3476 : // RHSSign -> RHS >= 0
3477 : // SumSign -> Sum >= 0
3478 : //
3479 : // Add:
3480 : // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3481 : // Sub:
3482 : // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3483 35 : SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3484 35 : SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3485 35 : SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3486 35 : Node->getOpcode() == ISD::SADDO ?
3487 53 : ISD::SETEQ : ISD::SETNE);
3488 :
3489 35 : SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3490 35 : SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3491 :
3492 70 : SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3493 35 : Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3494 : break;
3495 : }
3496 24 : case ISD::UADDO:
3497 : case ISD::USUBO: {
3498 24 : SDValue LHS = Node->getOperand(0);
3499 24 : SDValue RHS = Node->getOperand(1);
3500 : bool IsAdd = Node->getOpcode() == ISD::UADDO;
3501 : // If ADD/SUBCARRY is legal, use that instead.
3502 24 : unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
3503 48 : if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
3504 4 : SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
3505 2 : SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
3506 6 : { LHS, RHS, CarryIn });
3507 2 : Results.push_back(SDValue(NodeCarry.getNode(), 0));
3508 2 : Results.push_back(SDValue(NodeCarry.getNode(), 1));
3509 : break;
3510 : }
3511 :
3512 22 : SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
3513 33 : LHS.getValueType(), LHS, RHS);
3514 22 : Results.push_back(Sum);
3515 :
3516 44 : EVT ResultType = Node->getValueType(1);
3517 22 : EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3518 22 : ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3519 22 : SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3520 :
3521 22 : Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3522 22 : break;
3523 : }
3524 54 : case ISD::UMULO:
3525 : case ISD::SMULO: {
3526 54 : EVT VT = Node->getValueType(0);
3527 54 : EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3528 54 : SDValue LHS = Node->getOperand(0);
3529 54 : SDValue RHS = Node->getOperand(1);
3530 54 : SDValue BottomHalf;
3531 54 : SDValue TopHalf;
3532 : static const unsigned Ops[2][3] =
3533 : { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3534 : { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3535 54 : bool isSigned = Node->getOpcode() == ISD::SMULO;
3536 54 : if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3537 22 : BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3538 22 : TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3539 43 : } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3540 54 : BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3541 27 : RHS);
3542 27 : TopHalf = BottomHalf.getValue(1);
3543 16 : } else if (TLI.isTypeLegal(WideVT)) {
3544 0 : LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3545 0 : RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3546 0 : Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3547 0 : BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3548 0 : DAG.getIntPtrConstant(0, dl));
3549 0 : TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3550 0 : DAG.getIntPtrConstant(1, dl));
3551 : } else {
3552 : // We can fall back to a libcall with an illegal type for the MUL if we
3553 : // have a libcall big enough.
3554 : // Also, we can fall back to a division in some cases, but that's a big
3555 : // performance hit in the general case.
3556 : RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3557 : if (WideVT == MVT::i16)
3558 : LC = RTLIB::MUL_I16;
3559 : else if (WideVT == MVT::i32)
3560 : LC = RTLIB::MUL_I32;
3561 : else if (WideVT == MVT::i64)
3562 : LC = RTLIB::MUL_I64;
3563 : else if (WideVT == MVT::i128)
3564 : LC = RTLIB::MUL_I128;
3565 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
3566 :
3567 : SDValue HiLHS;
3568 : SDValue HiRHS;
3569 16 : if (isSigned) {
3570 : // The high part is obtained by SRA'ing all but one of the bits of low
3571 : // part.
3572 2 : unsigned LoSize = VT.getSizeInBits();
3573 2 : HiLHS =
3574 2 : DAG.getNode(ISD::SRA, dl, VT, LHS,
3575 2 : DAG.getConstant(LoSize - 1, dl,
3576 4 : TLI.getPointerTy(DAG.getDataLayout())));
3577 2 : HiRHS =
3578 2 : DAG.getNode(ISD::SRA, dl, VT, RHS,
3579 : DAG.getConstant(LoSize - 1, dl,
3580 4 : TLI.getPointerTy(DAG.getDataLayout())));
3581 : } else {
3582 14 : HiLHS = DAG.getConstant(0, dl, VT);
3583 14 : HiRHS = DAG.getConstant(0, dl, VT);
3584 : }
3585 :
3586 : // Here we're passing the 2 arguments explicitly as 4 arguments that are
3587 : // pre-lowered to the correct types. This all depends upon WideVT not
3588 : // being a legal type for the architecture and thus has to be split to
3589 : // two arguments.
3590 : SDValue Ret;
3591 16 : if(DAG.getDataLayout().isLittleEndian()) {
3592 : // Halves of WideVT are packed into registers in different order
3593 : // depending on platform endianness. This is usually handled by
3594 : // the C calling convention, but we can't defer to it in
3595 : // the legalizer.
3596 16 : SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3597 16 : Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3598 : } else {
3599 0 : SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3600 0 : Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3601 : }
3602 : assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
3603 : "Ret value is a collection of constituent nodes holding result.");
3604 16 : BottomHalf = Ret.getOperand(0);
3605 16 : TopHalf = Ret.getOperand(1);
3606 : }
3607 :
3608 54 : if (isSigned) {
3609 16 : Tmp1 = DAG.getConstant(
3610 8 : VT.getSizeInBits() - 1, dl,
3611 8 : TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3612 16 : Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3613 8 : TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3614 8 : ISD::SETNE);
3615 : } else {
3616 46 : TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3617 46 : DAG.getConstant(0, dl, VT), ISD::SETNE);
3618 : }
3619 :
3620 : // Truncate the result if SetCC returns a larger type than needed.
3621 54 : EVT RType = Node->getValueType(1);
3622 54 : if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3623 2 : TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3624 :
3625 : assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
3626 : "Unexpected result type for S/UMULO legalization");
3627 :
3628 54 : Results.push_back(BottomHalf);
3629 54 : Results.push_back(TopHalf);
3630 : break;
3631 : }
3632 23 : case ISD::BUILD_PAIR: {
3633 23 : EVT PairTy = Node->getValueType(0);
3634 69 : Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3635 69 : Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3636 23 : Tmp2 = DAG.getNode(
3637 : ISD::SHL, dl, PairTy, Tmp2,
3638 23 : DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3639 23 : TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3640 46 : Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3641 : break;
3642 : }
3643 216 : case ISD::SELECT:
3644 216 : Tmp1 = Node->getOperand(0);
3645 216 : Tmp2 = Node->getOperand(1);
3646 216 : Tmp3 = Node->getOperand(2);
3647 432 : if (Tmp1.getOpcode() == ISD::SETCC) {
3648 156 : Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3649 : Tmp2, Tmp3,
3650 312 : cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3651 : } else {
3652 60 : Tmp1 = DAG.getSelectCC(dl, Tmp1,
3653 : DAG.getConstant(0, dl, Tmp1.getValueType()),
3654 120 : Tmp2, Tmp3, ISD::SETNE);
3655 : }
3656 216 : Results.push_back(Tmp1);
3657 216 : break;
3658 3157 : case ISD::BR_JT: {
3659 3157 : SDValue Chain = Node->getOperand(0);
3660 3157 : SDValue Table = Node->getOperand(1);
3661 3157 : SDValue Index = Node->getOperand(2);
3662 :
3663 3157 : const DataLayout &TD = DAG.getDataLayout();
3664 : EVT PTy = TLI.getPointerTy(TD);
3665 :
3666 : unsigned EntrySize =
3667 3157 : DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3668 :
3669 : // For power-of-two jumptable entry sizes convert multiplication to a shift.
3670 : // This transformation needs to be done here since otherwise the MIPS
3671 : // backend will end up emitting a three instruction multiply sequence
3672 : // instead of a single shift and MSP430 will call a runtime function.
3673 : if (llvm::isPowerOf2_32(EntrySize))
3674 3157 : Index = DAG.getNode(
3675 : ISD::SHL, dl, Index.getValueType(), Index,
3676 3157 : DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3677 : else
3678 0 : Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3679 0 : DAG.getConstant(EntrySize, dl, Index.getValueType()));
3680 3157 : SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3681 3157 : Index, Table);
3682 :
3683 3157 : EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3684 3157 : SDValue LD = DAG.getExtLoad(
3685 : ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3686 3157 : MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3687 3157 : Addr = LD;
3688 3157 : if (TLI.isJumpTableRelative()) {
3689 : // For PIC, the sequence is:
3690 : // BRIND(load(Jumptable + index) + RelocBase)
3691 : // RelocBase can be JumpTable, GOT or some sort of global base.
3692 143 : Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3693 143 : TLI.getPICJumpTableRelocBase(Table, DAG));
3694 : }
3695 :
3696 3157 : Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3697 3157 : Results.push_back(Tmp1);
3698 : break;
3699 : }
3700 505 : case ISD::BRCOND:
3701 : // Expand brcond's setcc into its constituent parts and create a BR_CC
3702 : // Node.
3703 505 : Tmp1 = Node->getOperand(0);
3704 505 : Tmp2 = Node->getOperand(1);
3705 1010 : if (Tmp2.getOpcode() == ISD::SETCC) {
3706 1 : Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3707 : Tmp1, Tmp2.getOperand(2),
3708 : Tmp2.getOperand(0), Tmp2.getOperand(1),
3709 1 : Node->getOperand(2));
3710 : } else {
3711 : // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3712 504 : if (Tmp2.isUndef() ||
3713 : (Tmp2.getOpcode() == ISD::AND &&
3714 93 : isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3715 93 : cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3716 93 : Tmp3 = Tmp2;
3717 : else
3718 411 : Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3719 822 : DAG.getConstant(1, dl, Tmp2.getValueType()));
3720 1008 : Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3721 504 : DAG.getCondCode(ISD::SETNE), Tmp3,
3722 : DAG.getConstant(0, dl, Tmp3.getValueType()),
3723 1512 : Node->getOperand(2));
3724 : }
3725 505 : Results.push_back(Tmp1);
3726 505 : break;
3727 2672 : case ISD::SETCC: {
3728 2672 : Tmp1 = Node->getOperand(0);
3729 2672 : Tmp2 = Node->getOperand(1);
3730 2672 : Tmp3 = Node->getOperand(2);
3731 5344 : bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3732 : Tmp3, NeedInvert, dl);
3733 :
3734 2672 : if (Legalized) {
3735 : // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3736 : // condition code, create a new SETCC node.
3737 549 : if (Tmp3.getNode())
3738 276 : Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3739 552 : Tmp1, Tmp2, Tmp3);
3740 :
3741 : // If we expanded the SETCC by inverting the condition code, then wrap
3742 : // the existing SETCC in a NOT to restore the intended condition.
3743 549 : if (NeedInvert)
3744 210 : Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3745 :
3746 549 : Results.push_back(Tmp1);
3747 549 : break;
3748 : }
3749 :
3750 : // Otherwise, SETCC for the given comparison type must be completely
3751 : // illegal; expand it into a SELECT_CC.
3752 2123 : EVT VT = Node->getValueType(0);
3753 : int TrueValue;
3754 4246 : switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3755 762 : case TargetLowering::ZeroOrOneBooleanContent:
3756 : case TargetLowering::UndefinedBooleanContent:
3757 : TrueValue = 1;
3758 762 : break;
3759 1361 : case TargetLowering::ZeroOrNegativeOneBooleanContent:
3760 : TrueValue = -1;
3761 1361 : break;
3762 : }
3763 4246 : Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3764 2123 : DAG.getConstant(TrueValue, dl, VT),
3765 : DAG.getConstant(0, dl, VT),
3766 2123 : Tmp3);
3767 2123 : Results.push_back(Tmp1);
3768 2123 : break;
3769 : }
3770 1315 : case ISD::SELECT_CC: {
3771 1315 : Tmp1 = Node->getOperand(0); // LHS
3772 1315 : Tmp2 = Node->getOperand(1); // RHS
3773 1315 : Tmp3 = Node->getOperand(2); // True
3774 1315 : Tmp4 = Node->getOperand(3); // False
3775 1315 : EVT VT = Node->getValueType(0);
3776 1315 : SDValue CC = Node->getOperand(4);
3777 1315 : ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3778 :
3779 1315 : if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3780 : // If the condition code is legal, then we need to expand this
3781 : // node using SETCC and SELECT.
3782 1010 : EVT CmpVT = Tmp1.getValueType();
3783 : assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3784 : "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3785 : "expanded.");
3786 : EVT CCVT =
3787 1010 : TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3788 2020 : SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3789 1010 : Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3790 : break;
3791 : }
3792 :
3793 : // SELECT_CC is legal, so the condition code must not be.
3794 : bool Legalized = false;
3795 : // Try to legalize by inverting the condition. This is for targets that
3796 : // might support an ordered version of a condition, but not the unordered
3797 : // version (or vice versa).
3798 305 : ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3799 305 : Tmp1.getValueType().isInteger());
3800 305 : if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3801 : // Use the new condition code and swap true and false
3802 : Legalized = true;
3803 257 : Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3804 : } else {
3805 : // If The inverse is not legal, then try to swap the arguments using
3806 : // the inverse condition code.
3807 48 : ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3808 48 : if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3809 : // The swapped inverse condition is legal, so swap true and false,
3810 : // lhs and rhs.
3811 : Legalized = true;
3812 13 : Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3813 : }
3814 : }
3815 :
3816 : if (!Legalized) {
3817 35 : Legalized = LegalizeSetCCCondCode(
3818 : getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3819 : dl);
3820 :
3821 : assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3822 :
3823 : // If we expanded the SETCC by inverting the condition code, then swap
3824 : // the True/False operands to match.
3825 35 : if (NeedInvert)
3826 : std::swap(Tmp3, Tmp4);
3827 :
3828 : // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3829 : // condition code, create a new SELECT_CC node.
3830 35 : if (CC.getNode()) {
3831 13 : Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3832 26 : Tmp1, Tmp2, Tmp3, Tmp4, CC);
3833 : } else {
3834 44 : Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3835 22 : CC = DAG.getCondCode(ISD::SETNE);
3836 22 : Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3837 44 : Tmp2, Tmp3, Tmp4, CC);
3838 : }
3839 : }
3840 305 : Results.push_back(Tmp1);
3841 305 : break;
3842 : }
3843 9 : case ISD::BR_CC: {
3844 9 : Tmp1 = Node->getOperand(0); // Chain
3845 9 : Tmp2 = Node->getOperand(2); // LHS
3846 9 : Tmp3 = Node->getOperand(3); // RHS
3847 9 : Tmp4 = Node->getOperand(1); // CC
3848 :
3849 18 : bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3850 : Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3851 : (void)Legalized;
3852 : assert(Legalized && "Can't legalize BR_CC with legal condition!");
3853 :
3854 : // If we expanded the SETCC by inverting the condition code, then wrap
3855 : // the existing SETCC in a NOT to restore the intended condition.
3856 9 : if (NeedInvert)
3857 0 : Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3858 :
3859 : // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3860 : // node.
3861 9 : if (Tmp4.getNode()) {
3862 0 : Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3863 0 : Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3864 : } else {
3865 18 : Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3866 9 : Tmp4 = DAG.getCondCode(ISD::SETNE);
3867 9 : Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3868 27 : Tmp2, Tmp3, Node->getOperand(4));
3869 : }
3870 9 : Results.push_back(Tmp1);
3871 9 : break;
3872 : }
3873 26205 : case ISD::BUILD_VECTOR:
3874 26205 : Results.push_back(ExpandBUILD_VECTOR(Node));
3875 26205 : break;
3876 38 : case ISD::SRA:
3877 : case ISD::SRL:
3878 : case ISD::SHL: {
3879 : // Scalarize vector SRA/SRL/SHL.
3880 76 : EVT VT = Node->getValueType(0);
3881 : assert(VT.isVector() && "Unable to legalize non-vector shift");
3882 : assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3883 : unsigned NumElem = VT.getVectorNumElements();
3884 :
3885 : SmallVector<SDValue, 8> Scalars;
3886 142 : for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3887 104 : SDValue Ex = DAG.getNode(
3888 104 : ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3889 104 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3890 104 : SDValue Sh = DAG.getNode(
3891 104 : ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3892 104 : DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3893 208 : Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3894 208 : VT.getScalarType(), Ex, Sh));
3895 : }
3896 :
3897 114 : SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3898 38 : ReplaceNode(SDValue(Node, 0), Result);
3899 : break;
3900 : }
3901 4 : case ISD::ROTL:
3902 : case ISD::ROTR: {
3903 : bool IsLeft = Node->getOpcode() == ISD::ROTL;
3904 4 : SDValue Op0 = Node->getOperand(0), Op1 = Node->getOperand(1);
3905 8 : EVT ResVT = Node->getValueType(0);
3906 4 : EVT OpVT = Op0.getValueType();
3907 : assert(OpVT == ResVT &&
3908 : "The result and the operand types of rotate should match");
3909 4 : EVT ShVT = Op1.getValueType();
3910 4 : SDValue Width = DAG.getConstant(OpVT.getScalarSizeInBits(), dl, ShVT);
3911 :
3912 : // If a rotate in the other direction is legal, use it.
3913 4 : unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
3914 4 : if (TLI.isOperationLegal(RevRot, ResVT)) {
3915 0 : SDValue Sub = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3916 0 : Results.push_back(DAG.getNode(RevRot, dl, ResVT, Op0, Sub));
3917 : break;
3918 : }
3919 :
3920 : // Otherwise,
3921 : // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
3922 : // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
3923 : //
3924 : assert(isPowerOf2_32(OpVT.getScalarSizeInBits()) &&
3925 : "Expecting the type bitwidth to be a power of 2");
3926 4 : unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
3927 4 : unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
3928 4 : SDValue Width1 = DAG.getNode(ISD::SUB, dl, ShVT,
3929 4 : Width, DAG.getConstant(1, dl, ShVT));
3930 8 : SDValue NegOp1 = DAG.getNode(ISD::SUB, dl, ShVT, Width, Op1);
3931 8 : SDValue And0 = DAG.getNode(ISD::AND, dl, ShVT, Op1, Width1);
3932 8 : SDValue And1 = DAG.getNode(ISD::AND, dl, ShVT, NegOp1, Width1);
3933 :
3934 4 : SDValue Or = DAG.getNode(ISD::OR, dl, ResVT,
3935 4 : DAG.getNode(ShOpc, dl, ResVT, Op0, And0),
3936 4 : DAG.getNode(HsOpc, dl, ResVT, Op0, And1));
3937 4 : Results.push_back(Or);
3938 4 : break;
3939 : }
3940 :
3941 : case ISD::GLOBAL_OFFSET_TABLE:
3942 : case ISD::GlobalAddress:
3943 : case ISD::GlobalTLSAddress:
3944 : case ISD::ExternalSymbol:
3945 : case ISD::ConstantPool:
3946 : case ISD::JumpTable:
3947 : case ISD::INTRINSIC_W_CHAIN:
3948 : case ISD::INTRINSIC_WO_CHAIN:
3949 : case ISD::INTRINSIC_VOID:
3950 : // FIXME: Custom lowering for these operations shouldn't return null!
3951 : break;
3952 : }
3953 :
3954 : // Replace the original node with the legalized result.
3955 345107 : if (Results.empty()) {
3956 : LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3957 : return false;
3958 : }
3959 :
3960 : LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3961 296594 : ReplaceNode(Node, Results.data());
3962 296594 : return true;
3963 : }
3964 :
3965 48585 : void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3966 : LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3967 : SmallVector<SDValue, 8> Results;
3968 : SDLoc dl(Node);
3969 : // FIXME: Check flags on the node to see if we can use a finite call.
3970 48585 : bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
3971 48585 : unsigned Opc = Node->getOpcode();
3972 48585 : switch (Opc) {
3973 : case ISD::ATOMIC_FENCE: {
3974 : // If the target didn't lower this, lower it to '__sync_synchronize()' call
3975 : // FIXME: handle "fence singlethread" more efficiently.
3976 : TargetLowering::ArgListTy Args;
3977 :
3978 26 : TargetLowering::CallLoweringInfo CLI(DAG);
3979 : CLI.setDebugLoc(dl)
3980 13 : .setChain(Node->getOperand(0))
3981 : .setLibCallee(
3982 13 : CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3983 13 : DAG.getExternalSymbol("__sync_synchronize",
3984 : TLI.getPointerTy(DAG.getDataLayout())),
3985 26 : std::move(Args));
3986 :
3987 13 : std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3988 :
3989 13 : Results.push_back(CallResult.second);
3990 : break;
3991 : }
3992 : // By default, atomic intrinsics are marked Legal and lowered. Targets
3993 : // which don't support them directly, however, may want libcalls, in which
3994 : // case they mark them Expand, and we get here.
3995 : case ISD::ATOMIC_SWAP:
3996 : case ISD::ATOMIC_LOAD_ADD:
3997 : case ISD::ATOMIC_LOAD_SUB:
3998 : case ISD::ATOMIC_LOAD_AND:
3999 : case ISD::ATOMIC_LOAD_CLR:
4000 : case ISD::ATOMIC_LOAD_OR:
4001 : case ISD::ATOMIC_LOAD_XOR:
4002 : case ISD::ATOMIC_LOAD_NAND:
4003 : case ISD::ATOMIC_LOAD_MIN:
4004 : case ISD::ATOMIC_LOAD_MAX:
4005 : case ISD::ATOMIC_LOAD_UMIN:
4006 : case ISD::ATOMIC_LOAD_UMAX:
4007 : case ISD::ATOMIC_CMP_SWAP: {
4008 487 : MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4009 487 : RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
4010 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
4011 :
4012 487 : std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
4013 487 : Results.push_back(Tmp.first);
4014 487 : Results.push_back(Tmp.second);
4015 : break;
4016 : }
4017 : case ISD::TRAP: {
4018 : // If this operation is not supported, lower it to 'abort()' call
4019 : TargetLowering::ArgListTy Args;
4020 0 : TargetLowering::CallLoweringInfo CLI(DAG);
4021 : CLI.setDebugLoc(dl)
4022 0 : .setChain(Node->getOperand(0))
4023 0 : .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4024 0 : DAG.getExternalSymbol(
4025 : "abort", TLI.getPointerTy(DAG.getDataLayout())),
4026 0 : std::move(Args));
4027 0 : std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4028 :
4029 0 : Results.push_back(CallResult.second);
4030 : break;
4031 : }
4032 46 : case ISD::FMINNUM:
4033 46 : Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4034 : RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4035 46 : RTLIB::FMIN_PPCF128));
4036 46 : break;
4037 53 : case ISD::FMAXNUM:
4038 53 : Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4039 : RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4040 53 : RTLIB::FMAX_PPCF128));
4041 53 : break;
4042 3 : case ISD::FSQRT:
4043 : case ISD::STRICT_FSQRT:
4044 3 : Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4045 : RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4046 3 : RTLIB::SQRT_PPCF128));
4047 3 : break;
4048 2 : case ISD::FCBRT:
4049 2 : Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4050 : RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4051 2 : RTLIB::CBRT_PPCF128));
4052 2 : break;
4053 199 : case ISD::FSIN:
4054 : case ISD::STRICT_FSIN:
4055 199 : Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4056 : RTLIB::SIN_F80, RTLIB::SIN_F128,
4057 199 : RTLIB::SIN_PPCF128));
4058 199 : break;
4059 132 : case ISD::FCOS:
4060 : case ISD::STRICT_FCOS:
4061 132 : Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4062 : RTLIB::COS_F80, RTLIB::COS_F128,
4063 132 : RTLIB::COS_PPCF128));
4064 132 : break;
4065 34 : case ISD::FSINCOS:
4066 : // Expand into sincos libcall.
4067 34 : ExpandSinCosLibCall(Node, Results);
4068 34 : break;
4069 104 : case ISD::FLOG:
4070 : case ISD::STRICT_FLOG:
4071 104 : if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
4072 4 : Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
4073 : RTLIB::LOG_FINITE_F64,
4074 : RTLIB::LOG_FINITE_F80,
4075 : RTLIB::LOG_FINITE_F128,
4076 4 : RTLIB::LOG_FINITE_PPCF128));
4077 : else
4078 100 : Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
4079 : RTLIB::LOG_F80, RTLIB::LOG_F128,
4080 100 : RTLIB::LOG_PPCF128));
4081 : break;
4082 106 : case ISD::FLOG2:
4083 : case ISD::STRICT_FLOG2:
4084 106 : if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
4085 4 : Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
4086 : RTLIB::LOG2_FINITE_F64,
4087 : RTLIB::LOG2_FINITE_F80,
4088 : RTLIB::LOG2_FINITE_F128,
4089 4 : RTLIB::LOG2_FINITE_PPCF128));
4090 : else
4091 102 : Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
4092 : RTLIB::LOG2_F80, RTLIB::LOG2_F128,
4093 102 : RTLIB::LOG2_PPCF128));
4094 : break;
4095 108 : case ISD::FLOG10:
4096 : case ISD::STRICT_FLOG10:
4097 108 : if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
4098 4 : Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
4099 : RTLIB::LOG10_FINITE_F64,
4100 : RTLIB::LOG10_FINITE_F80,
4101 : RTLIB::LOG10_FINITE_F128,
4102 4 : RTLIB::LOG10_FINITE_PPCF128));
4103 : else
4104 104 : Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
4105 : RTLIB::LOG10_F80, RTLIB::LOG10_F128,
4106 104 : RTLIB::LOG10_PPCF128));
4107 : break;
4108 119 : case ISD::FEXP:
4109 : case ISD::STRICT_FEXP:
4110 119 : if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
4111 4 : Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
4112 : RTLIB::EXP_FINITE_F64,
4113 : RTLIB::EXP_FINITE_F80,
4114 : RTLIB::EXP_FINITE_F128,
4115 4 : RTLIB::EXP_FINITE_PPCF128));
4116 : else
4117 115 : Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
4118 : RTLIB::EXP_F80, RTLIB::EXP_F128,
4119 115 : RTLIB::EXP_PPCF128));
4120 : break;
4121 108 : case ISD::FEXP2:
4122 : case ISD::STRICT_FEXP2:
4123 108 : if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
4124 4 : Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
4125 : RTLIB::EXP2_FINITE_F64,
4126 : RTLIB::EXP2_FINITE_F80,
4127 : RTLIB::EXP2_FINITE_F128,
4128 4 : RTLIB::EXP2_FINITE_PPCF128));
4129 : else
4130 104 : Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4131 : RTLIB::EXP2_F80, RTLIB::EXP2_F128,
4132 104 : RTLIB::EXP2_PPCF128));
4133 : break;
4134 38 : case ISD::FTRUNC:
4135 38 : Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4136 : RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4137 38 : RTLIB::TRUNC_PPCF128));
4138 38 : break;
4139 71 : case ISD::FFLOOR:
4140 71 : Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4141 : RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4142 71 : RTLIB::FLOOR_PPCF128));
4143 71 : break;
4144 1965 : case ISD::FCEIL:
4145 1965 : Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4146 : RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4147 1965 : RTLIB::CEIL_PPCF128));
4148 1965 : break;
4149 46 : case ISD::FRINT:
4150 : case ISD::STRICT_FRINT:
4151 46 : Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4152 : RTLIB::RINT_F80, RTLIB::RINT_F128,
4153 46 : RTLIB::RINT_PPCF128));
4154 46 : break;
4155 63 : case ISD::FNEARBYINT:
4156 : case ISD::STRICT_FNEARBYINT:
4157 63 : Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4158 : RTLIB::NEARBYINT_F64,
4159 : RTLIB::NEARBYINT_F80,
4160 : RTLIB::NEARBYINT_F128,
4161 63 : RTLIB::NEARBYINT_PPCF128));
4162 63 : break;
4163 21 : case ISD::FROUND:
4164 21 : Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4165 : RTLIB::ROUND_F64,
4166 : RTLIB::ROUND_F80,
4167 : RTLIB::ROUND_F128,
4168 21 : RTLIB::ROUND_PPCF128));
4169 21 : break;
4170 134 : case ISD::FPOWI:
4171 : case ISD::STRICT_FPOWI:
4172 134 : Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4173 : RTLIB::POWI_F80, RTLIB::POWI_F128,
4174 134 : RTLIB::POWI_PPCF128));
4175 134 : break;
4176 159 : case ISD::FPOW:
4177 : case ISD::STRICT_FPOW:
4178 159 : if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
4179 4 : Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
4180 : RTLIB::POW_FINITE_F64,
4181 : RTLIB::POW_FINITE_F80,
4182 : RTLIB::POW_FINITE_F128,
4183 4 : RTLIB::POW_FINITE_PPCF128));
4184 : else
4185 155 : Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4186 : RTLIB::POW_F80, RTLIB::POW_F128,
4187 155 : RTLIB::POW_PPCF128));
4188 : break;
4189 9 : case ISD::FDIV:
4190 9 : Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4191 : RTLIB::DIV_F80, RTLIB::DIV_F128,
4192 9 : RTLIB::DIV_PPCF128));
4193 9 : break;
4194 130 : case ISD::FREM:
4195 : case ISD::STRICT_FREM:
4196 130 : Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4197 : RTLIB::REM_F80, RTLIB::REM_F128,
4198 130 : RTLIB::REM_PPCF128));
4199 130 : break;
4200 268 : case ISD::FMA:
4201 : case ISD::STRICT_FMA:
4202 268 : Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4203 : RTLIB::FMA_F80, RTLIB::FMA_F128,
4204 268 : RTLIB::FMA_PPCF128));
4205 268 : break;
4206 47 : case ISD::FADD:
4207 47 : Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4208 : RTLIB::ADD_F80, RTLIB::ADD_F128,
4209 47 : RTLIB::ADD_PPCF128));
4210 47 : break;
4211 131 : case ISD::FMUL:
4212 131 : Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4213 : RTLIB::MUL_F80, RTLIB::MUL_F128,
4214 131 : RTLIB::MUL_PPCF128));
4215 131 : break;
4216 : case ISD::FP16_TO_FP:
4217 357 : if (Node->getValueType(0) == MVT::f32) {
4218 357 : Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4219 : }
4220 : break;
4221 : case ISD::FP_TO_FP16: {
4222 : RTLIB::Libcall LC =
4223 940 : RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4224 : assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4225 470 : Results.push_back(ExpandLibCall(LC, Node, false));
4226 470 : break;
4227 : }
4228 75 : case ISD::FSUB:
4229 75 : Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4230 : RTLIB::SUB_F80, RTLIB::SUB_F128,
4231 75 : RTLIB::SUB_PPCF128));
4232 75 : break;
4233 22 : case ISD::SREM:
4234 22 : Results.push_back(ExpandIntLibCall(Node, true,
4235 : RTLIB::SREM_I8,
4236 : RTLIB::SREM_I16, RTLIB::SREM_I32,
4237 22 : RTLIB::SREM_I64, RTLIB::SREM_I128));
4238 22 : break;
4239 16 : case ISD::UREM:
4240 16 : Results.push_back(ExpandIntLibCall(Node, false,
4241 : RTLIB::UREM_I8,
4242 : RTLIB::UREM_I16, RTLIB::UREM_I32,
4243 16 : RTLIB::UREM_I64, RTLIB::UREM_I128));
4244 16 : break;
4245 48 : case ISD::SDIV:
4246 48 : Results.push_back(ExpandIntLibCall(Node, true,
4247 : RTLIB::SDIV_I8,
4248 : RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4249 48 : RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4250 48 : break;
4251 21 : case ISD::UDIV:
4252 21 : Results.push_back(ExpandIntLibCall(Node, false,
4253 : RTLIB::UDIV_I8,
4254 : RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4255 21 : RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4256 21 : break;
4257 14 : case ISD::SDIVREM:
4258 : case ISD::UDIVREM:
4259 : // Expand into divrem libcall
4260 14 : ExpandDivRemLibCall(Node, Results);
4261 14 : break;
4262 15 : case ISD::MUL:
4263 15 : Results.push_back(ExpandIntLibCall(Node, false,
4264 : RTLIB::MUL_I8,
4265 : RTLIB::MUL_I16, RTLIB::MUL_I32,
4266 15 : RTLIB::MUL_I64, RTLIB::MUL_I128));
4267 15 : break;
4268 : case ISD::CTLZ_ZERO_UNDEF:
4269 : switch (Node->getSimpleValueType(0).SimpleTy) {
4270 0 : default:
4271 0 : llvm_unreachable("LibCall explicitly requested, but not available");
4272 1 : case MVT::i32:
4273 1 : Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4274 1 : break;
4275 0 : case MVT::i64:
4276 0 : Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4277 0 : break;
4278 0 : case MVT::i128:
4279 0 : Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4280 0 : break;
4281 : }
4282 : break;
4283 : }
4284 :
4285 : // Replace the original node with the legalized result.
4286 48585 : if (!Results.empty()) {
4287 : LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4288 5635 : ReplaceNode(Node, Results.data());
4289 : } else
4290 : LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4291 48585 : }
4292 :
4293 : // Determine the vector type to use in place of an original scalar element when
4294 : // promoting equally sized vectors.
4295 0 : static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4296 : MVT EltVT, MVT NewEltVT) {
4297 0 : unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4298 0 : MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4299 : assert(TLI.isTypeLegal(MidVT) && "unexpected");
4300 0 : return MidVT;
4301 : }
4302 :
4303 16052 : void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4304 : LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4305 : SmallVector<SDValue, 8> Results;
4306 16052 : MVT OVT = Node->getSimpleValueType(0);
4307 16052 : if (Node->getOpcode() == ISD::UINT_TO_FP ||
4308 14923 : Node->getOpcode() == ISD::SINT_TO_FP ||
4309 14728 : Node->getOpcode() == ISD::SETCC ||
4310 20112 : Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4311 : Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4312 23992 : OVT = Node->getOperand(0).getSimpleValueType();
4313 : }
4314 32104 : if (Node->getOpcode() == ISD::BR_CC)
4315 2 : OVT = Node->getOperand(2).getSimpleValueType();
4316 16052 : MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4317 : SDLoc dl(Node);
4318 16052 : SDValue Tmp1, Tmp2, Tmp3;
4319 32104 : switch (Node->getOpcode()) {
4320 77 : case ISD::CTTZ:
4321 : case ISD::CTTZ_ZERO_UNDEF:
4322 : case ISD::CTLZ:
4323 : case ISD::CTLZ_ZERO_UNDEF:
4324 : case ISD::CTPOP:
4325 : // Zero extend the argument.
4326 231 : Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4327 77 : if (Node->getOpcode() == ISD::CTTZ) {
4328 : // The count is the same in the promoted type except if the original
4329 : // value was zero. This can be handled by setting the bit just off
4330 : // the top of the original type.
4331 : auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4332 3 : OVT.getSizeInBits());
4333 3 : Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4334 3 : DAG.getConstant(TopBit, dl, NVT));
4335 : }
4336 : // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4337 : // already the correct result.
4338 231 : Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4339 154 : if (Node->getOpcode() == ISD::CTLZ ||
4340 : Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4341 : // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4342 11 : Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4343 22 : DAG.getConstant(NVT.getSizeInBits() -
4344 11 : OVT.getSizeInBits(), dl, NVT));
4345 : }
4346 154 : Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4347 77 : break;
4348 3 : case ISD::BITREVERSE:
4349 : case ISD::BSWAP: {
4350 3 : unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4351 9 : Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4352 9 : Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4353 3 : Tmp1 = DAG.getNode(
4354 : ISD::SRL, dl, NVT, Tmp1,
4355 : DAG.getConstant(DiffBits, dl,
4356 3 : TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4357 :
4358 6 : Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4359 3 : break;
4360 : }
4361 255 : case ISD::FP_TO_UINT:
4362 : case ISD::FP_TO_SINT:
4363 255 : Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4364 510 : Node->getOpcode() == ISD::FP_TO_SINT, dl);
4365 255 : Results.push_back(Tmp1);
4366 255 : break;
4367 1129 : case ISD::UINT_TO_FP:
4368 : case ISD::SINT_TO_FP:
4369 1129 : Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4370 2258 : Node->getOpcode() == ISD::SINT_TO_FP, dl);
4371 1129 : Results.push_back(Tmp1);
4372 1129 : break;
4373 1 : case ISD::VAARG: {
4374 1 : SDValue Chain = Node->getOperand(0); // Get the chain.
4375 1 : SDValue Ptr = Node->getOperand(1); // Get the pointer.
4376 :
4377 : unsigned TruncOp;
4378 2 : if (OVT.isVector()) {
4379 : TruncOp = ISD::BITCAST;
4380 : } else {
4381 : assert(OVT.isInteger()
4382 : && "VAARG promotion is supported only for vectors or integer types");
4383 : TruncOp = ISD::TRUNCATE;
4384 : }
4385 :
4386 : // Perform the larger operation, then convert back
4387 1 : Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4388 2 : Node->getConstantOperandVal(3));
4389 1 : Chain = Tmp1.getValue(1);
4390 :
4391 2 : Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4392 :
4393 : // Modified the chain result - switch anything that used the old chain to
4394 : // use the new one.
4395 2 : DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4396 2 : DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4397 1 : if (UpdatedNodes) {
4398 0 : UpdatedNodes->insert(Tmp2.getNode());
4399 0 : UpdatedNodes->insert(Chain.getNode());
4400 : }
4401 1 : ReplacedNode(Node);
4402 : break;
4403 : }
4404 53 : case ISD::MUL:
4405 : case ISD::SDIV:
4406 : case ISD::SREM:
4407 : case ISD::UDIV:
4408 : case ISD::UREM:
4409 : case ISD::AND:
4410 : case ISD::OR:
4411 : case ISD::XOR: {
4412 : unsigned ExtOp, TruncOp;
4413 106 : if (OVT.isVector()) {
4414 : ExtOp = ISD::BITCAST;
4415 : TruncOp = ISD::BITCAST;
4416 : } else {
4417 : assert(OVT.isInteger() && "Cannot promote logic operation");
4418 :
4419 : switch (Node->getOpcode()) {
4420 : default:
4421 : ExtOp = ISD::ANY_EXTEND;
4422 : break;
4423 0 : case ISD::SDIV:
4424 : case ISD::SREM:
4425 : ExtOp = ISD::SIGN_EXTEND;
4426 0 : break;
4427 0 : case ISD::UDIV:
4428 : case ISD::UREM:
4429 : ExtOp = ISD::ZERO_EXTEND;
4430 0 : break;
4431 : }
4432 : TruncOp = ISD::TRUNCATE;
4433 : }
4434 : // Promote each of the values to the new type.
4435 159 : Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4436 159 : Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4437 : // Perform the larger operation, then convert back
4438 159 : Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4439 106 : Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4440 53 : break;
4441 : }
4442 0 : case ISD::UMUL_LOHI:
4443 : case ISD::SMUL_LOHI: {
4444 : // Promote to a multiply in a wider integer type.
4445 0 : unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4446 : : ISD::SIGN_EXTEND;
4447 0 : Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4448 0 : Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4449 0 : Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4450 :
4451 0 : auto &DL = DAG.getDataLayout();
4452 0 : unsigned OriginalSize = OVT.getScalarSizeInBits();
4453 0 : Tmp2 = DAG.getNode(
4454 : ISD::SRL, dl, NVT, Tmp1,
4455 0 : DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4456 0 : Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4457 0 : Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4458 0 : break;
4459 : }
4460 180 : case ISD::SELECT: {
4461 : unsigned ExtOp, TruncOp;
4462 540 : if (Node->getValueType(0).isVector() ||
4463 180 : Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4464 : ExtOp = ISD::BITCAST;
4465 : TruncOp = ISD::BITCAST;
4466 9 : } else if (Node->getValueType(0).isInteger()) {
4467 : ExtOp = ISD::ANY_EXTEND;
4468 : TruncOp = ISD::TRUNCATE;
4469 : } else {
4470 : ExtOp = ISD::FP_EXTEND;
4471 : TruncOp = ISD::FP_ROUND;
4472 : }
4473 180 : Tmp1 = Node->getOperand(0);
4474 : // Promote each of the values to the new type.
4475 360 : Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4476 540 : Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4477 : // Perform the larger operation, then round down.
4478 360 : Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4479 180 : if (TruncOp != ISD::FP_ROUND)
4480 525 : Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4481 : else
4482 5 : Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4483 5 : DAG.getIntPtrConstant(0, dl));
4484 180 : Results.push_back(Tmp1);
4485 180 : break;
4486 : }
4487 : case ISD::VECTOR_SHUFFLE: {
4488 571 : ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4489 :
4490 : // Cast the two input vectors.
4491 1713 : Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4492 1713 : Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4493 :
4494 : // Convert the shuffle mask to the right # elements.
4495 571 : Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4496 1142 : Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4497 571 : Results.push_back(Tmp1);
4498 : break;
4499 : }
4500 195 : case ISD::SETCC: {
4501 : unsigned ExtOp = ISD::FP_EXTEND;
4502 390 : if (NVT.isInteger()) {
4503 : ISD::CondCode CCCode =
4504 152 : cast<CondCodeSDNode>(Node->getOperand(2))->get();
4505 76 : ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4506 : }
4507 585 : Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4508 585 : Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4509 195 : Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4510 585 : Tmp1, Tmp2, Node->getOperand(2)));
4511 195 : break;
4512 : }
4513 1 : case ISD::BR_CC: {
4514 : unsigned ExtOp = ISD::FP_EXTEND;
4515 2 : if (NVT.isInteger()) {
4516 : ISD::CondCode CCCode =
4517 0 : cast<CondCodeSDNode>(Node->getOperand(1))->get();
4518 0 : ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4519 : }
4520 3 : Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4521 3 : Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4522 1 : Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4523 : Node->getOperand(0), Node->getOperand(1),
4524 3 : Tmp1, Tmp2, Node->getOperand(4)));
4525 1 : break;
4526 : }
4527 194 : case ISD::FADD:
4528 : case ISD::FSUB:
4529 : case ISD::FMUL:
4530 : case ISD::FDIV:
4531 : case ISD::FREM:
4532 : case ISD::FMINNUM:
4533 : case ISD::FMAXNUM:
4534 : case ISD::FPOW:
4535 582 : Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4536 582 : Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4537 194 : Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4538 582 : Node->getFlags());
4539 388 : Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4540 194 : Tmp3, DAG.getIntPtrConstant(0, dl)));
4541 194 : break;
4542 25 : case ISD::FMA:
4543 75 : Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4544 75 : Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4545 75 : Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4546 25 : Results.push_back(
4547 25 : DAG.getNode(ISD::FP_ROUND, dl, OVT,
4548 25 : DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4549 25 : DAG.getIntPtrConstant(0, dl)));
4550 25 : break;
4551 12 : case ISD::FCOPYSIGN:
4552 : case ISD::FPOWI: {
4553 36 : Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4554 12 : Tmp2 = Node->getOperand(1);
4555 36 : Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4556 :
4557 : // fcopysign doesn't change anything but the sign bit, so
4558 : // (fp_round (fcopysign (fpext a), b))
4559 : // is as precise as
4560 : // (fp_round (fpext a))
4561 : // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4562 12 : const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4563 24 : Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4564 12 : Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4565 12 : break;
4566 : }
4567 175 : case ISD::FFLOOR:
4568 : case ISD::FCEIL:
4569 : case ISD::FRINT:
4570 : case ISD::FNEARBYINT:
4571 : case ISD::FROUND:
4572 : case ISD::FTRUNC:
4573 : case ISD::FNEG:
4574 : case ISD::FSQRT:
4575 : case ISD::FSIN:
4576 : case ISD::FCOS:
4577 : case ISD::FLOG:
4578 : case ISD::FLOG2:
4579 : case ISD::FLOG10:
4580 : case ISD::FABS:
4581 : case ISD::FEXP:
4582 : case ISD::FEXP2:
4583 525 : Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4584 525 : Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4585 350 : Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4586 175 : Tmp2, DAG.getIntPtrConstant(0, dl)));
4587 175 : break;
4588 398 : case ISD::BUILD_VECTOR: {
4589 398 : MVT EltVT = OVT.getVectorElementType();
4590 398 : MVT NewEltVT = NVT.getVectorElementType();
4591 :
4592 : // Handle bitcasts to a different vector type with the same total bit size
4593 : //
4594 : // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4595 : // =>
4596 : // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4597 :
4598 : assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4599 : "Invalid promote type for build_vector");
4600 : assert(NewEltVT.bitsLT(EltVT) && "not handled");
4601 :
4602 398 : MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4603 :
4604 : SmallVector<SDValue, 8> NewOps;
4605 1194 : for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4606 796 : SDValue Op = Node->getOperand(I);
4607 796 : NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4608 : }
4609 :
4610 : SDLoc SL(Node);
4611 796 : SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4612 796 : SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4613 398 : Results.push_back(CvtVec);
4614 : break;
4615 : }
4616 10668 : case ISD::EXTRACT_VECTOR_ELT: {
4617 10668 : MVT EltVT = OVT.getVectorElementType();
4618 10668 : MVT NewEltVT = NVT.getVectorElementType();
4619 :
4620 : // Handle bitcasts to a different vector type with the same total bit size.
4621 : //
4622 : // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4623 : // =>
4624 : // v4i32:castx = bitcast x:v2i64
4625 : //
4626 : // i64 = bitcast
4627 : // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4628 : // (i32 (extract_vector_elt castx, (2 * y + 1)))
4629 : //
4630 :
4631 : assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4632 : "Invalid promote type for extract_vector_elt");
4633 : assert(NewEltVT.bitsLT(EltVT) && "not handled");
4634 :
4635 10668 : MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4636 : unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4637 :
4638 10668 : SDValue Idx = Node->getOperand(1);
4639 10668 : EVT IdxVT = Idx.getValueType();
4640 : SDLoc SL(Node);
4641 10668 : SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4642 21336 : SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4643 :
4644 32004 : SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4645 :
4646 : SmallVector<SDValue, 8> NewOps;
4647 32004 : for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4648 21336 : SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4649 42672 : SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4650 :
4651 21336 : SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4652 21336 : CastVec, TmpIdx);
4653 21336 : NewOps.push_back(Elt);
4654 : }
4655 :
4656 21336 : SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4657 21336 : Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4658 : break;
4659 : }
4660 4 : case ISD::INSERT_VECTOR_ELT: {
4661 4 : MVT EltVT = OVT.getVectorElementType();
4662 4 : MVT NewEltVT = NVT.getVectorElementType();
4663 :
4664 : // Handle bitcasts to a different vector type with the same total bit size
4665 : //
4666 : // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4667 : // =>
4668 : // v4i32:castx = bitcast x:v2i64
4669 : // v2i32:casty = bitcast y:i64
4670 : //
4671 : // v2i64 = bitcast
4672 : // (v4i32 insert_vector_elt
4673 : // (v4i32 insert_vector_elt v4i32:castx,
4674 : // (extract_vector_elt casty, 0), 2 * z),
4675 : // (extract_vector_elt casty, 1), (2 * z + 1))
4676 :
4677 : assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4678 : "Invalid promote type for insert_vector_elt");
4679 : assert(NewEltVT.bitsLT(EltVT) && "not handled");
4680 :
4681 4 : MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4682 : unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4683 :
4684 4 : SDValue Val = Node->getOperand(1);
4685 4 : SDValue Idx = Node->getOperand(2);
4686 4 : EVT IdxVT = Idx.getValueType();
4687 : SDLoc SL(Node);
4688 :
4689 4 : SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4690 8 : SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4691 :
4692 12 : SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4693 8 : SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4694 :
4695 4 : SDValue NewVec = CastVec;
4696 12 : for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4697 8 : SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4698 16 : SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4699 :
4700 8 : SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4701 8 : CastVal, IdxOffset);
4702 :
4703 8 : NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4704 8 : NewVec, Elt, InEltIdx);
4705 : }
4706 :
4707 8 : Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4708 : break;
4709 : }
4710 1959 : case ISD::SCALAR_TO_VECTOR: {
4711 1959 : MVT EltVT = OVT.getVectorElementType();
4712 1959 : MVT NewEltVT = NVT.getVectorElementType();
4713 :
4714 : // Handle bitcasts to different vector type with the same total bit size.
4715 : //
4716 : // e.g. v2i64 = scalar_to_vector x:i64
4717 : // =>
4718 : // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4719 : //
4720 :
4721 1959 : MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4722 1959 : SDValue Val = Node->getOperand(0);
4723 : SDLoc SL(Node);
4724 :
4725 3918 : SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4726 3918 : SDValue Undef = DAG.getUNDEF(MidVT);
4727 :
4728 : SmallVector<SDValue, 8> NewElts;
4729 1959 : NewElts.push_back(CastVal);
4730 3918 : for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4731 1959 : NewElts.push_back(Undef);
4732 :
4733 3918 : SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4734 3918 : SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4735 1959 : Results.push_back(CvtVec);
4736 : break;
4737 : }
4738 : }
4739 :
4740 : // Replace the original node with the legalized result.
4741 16052 : if (!Results.empty()) {
4742 : LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
4743 15899 : ReplaceNode(Node, Results.data());
4744 : } else
4745 : LLVM_DEBUG(dbgs() << "Could not promote node\n");
4746 16052 : }
4747 :
4748 : /// This is the entry point for the file.
4749 1269116 : void SelectionDAG::Legalize() {
4750 1269116 : AssignTopologicalOrder();
4751 :
4752 : SmallPtrSet<SDNode *, 16> LegalizedNodes;
4753 : // Use a delete listener to remove nodes which were deleted during
4754 : // legalization from LegalizeNodes. This is needed to handle the situation
4755 : // where a new node is allocated by the object pool to the same address of a
4756 : // previously deleted node.
4757 : DAGNodeDeletedListener DeleteListener(
4758 : *this,
4759 1269116 : [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4760 :
4761 : SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4762 :
4763 : // Visit all the nodes. We start in topological order, so that we see
4764 : // nodes with their original operands intact. Legalization can produce
4765 : // new nodes which may themselves need to be legalized. Iterate until all
4766 : // nodes have been legalized.
4767 : while (true) {
4768 : bool AnyLegalized = false;
4769 101278302 : for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4770 : --NI;
4771 :
4772 : SDNode *N = &*NI;
4773 98149641 : if (N->use_empty() && N != getRoot().getNode()) {
4774 : ++NI;
4775 857309 : DeleteNode(N);
4776 857309 : continue;
4777 : }
4778 :
4779 97292332 : if (LegalizedNodes.insert(N).second) {
4780 : AnyLegalized = true;
4781 39291485 : Legalizer.LegalizeOp(N);
4782 :
4783 39291478 : if (N->use_empty() && N != getRoot().getNode()) {
4784 : ++NI;
4785 1774517 : DeleteNode(N);
4786 : }
4787 : }
4788 : }
4789 3128661 : if (!AnyLegalized)
4790 : break;
4791 :
4792 : }
4793 :
4794 : // Remove dead nodes now.
4795 1269111 : RemoveDeadNodes();
4796 1269111 : }
4797 :
4798 39041762 : bool SelectionDAG::LegalizeOp(SDNode *N,
4799 : SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4800 : SmallPtrSet<SDNode *, 16> LegalizedNodes;
4801 : SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4802 :
4803 : // Directly insert the node in question, and legalize it. This will recurse
4804 : // as needed through operands.
4805 39041762 : LegalizedNodes.insert(N);
4806 39041762 : Legalizer.LegalizeOp(N);
4807 :
4808 39041762 : return LegalizedNodes.count(N);
4809 : }
|
