Line data Source code
1 : //===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file defines the DAGTypeLegalizer class. This is a private interface
11 : // shared between the code that implements the SelectionDAG::LegalizeTypes
12 : // method.
13 : //
14 : //===----------------------------------------------------------------------===//
15 :
16 : #ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
17 : #define LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
18 :
19 : #include "llvm/ADT/DenseMap.h"
20 : #include "llvm/CodeGen/SelectionDAG.h"
21 : #include "llvm/CodeGen/TargetLowering.h"
22 : #include "llvm/Support/Compiler.h"
23 : #include "llvm/Support/Debug.h"
24 :
25 : namespace llvm {
26 :
27 : //===----------------------------------------------------------------------===//
28 : /// This takes an arbitrary SelectionDAG as input and hacks on it until only
29 : /// value types the target machine can handle are left. This involves promoting
30 : /// small sizes to large sizes or splitting up large values into small values.
31 : ///
32 : class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
33 : const TargetLowering &TLI;
34 : SelectionDAG &DAG;
35 : public:
36 : /// This pass uses the NodeId on the SDNodes to hold information about the
37 : /// state of the node. The enum has all the values.
38 : enum NodeIdFlags {
39 : /// All operands have been processed, so this node is ready to be handled.
40 : ReadyToProcess = 0,
41 :
42 : /// This is a new node, not before seen, that was created in the process of
43 : /// legalizing some other node.
44 : NewNode = -1,
45 :
46 : /// This node's ID needs to be set to the number of its unprocessed
47 : /// operands.
48 : Unanalyzed = -2,
49 :
50 : /// This is a node that has already been processed.
51 : Processed = -3
52 :
53 : // 1+ - This is a node which has this many unprocessed operands.
54 : };
55 : private:
56 :
57 : /// This is a bitvector that contains two bits for each simple value type,
58 : /// where the two bits correspond to the LegalizeAction enum from
59 : /// TargetLowering. This can be queried with "getTypeAction(VT)".
60 : TargetLowering::ValueTypeActionImpl ValueTypeActions;
61 :
62 : /// Return how we should legalize values of this type.
63 0 : TargetLowering::LegalizeTypeAction getTypeAction(EVT VT) const {
64 96794344 : return TLI.getTypeAction(*DAG.getContext(), VT);
65 : }
66 :
67 : /// Return true if this type is legal on this target.
68 0 : bool isTypeLegal(EVT VT) const {
69 117963 : return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal;
70 : }
71 :
72 : /// Return true if this is a simple legal type.
73 0 : bool isSimpleLegalType(EVT VT) const {
74 5844 : return VT.isSimple() && TLI.isTypeLegal(VT);
75 : }
76 :
77 : /// Return true if this type can be passed in registers.
78 : /// For example, x86_64's f128, should to be legally in registers
79 : /// and only some operations converted to library calls or integer
80 : /// bitwise operations.
81 8214 : bool isLegalInHWReg(EVT VT) const {
82 8214 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
83 0 : return VT == NVT && isSimpleLegalType(VT);
84 : }
85 :
86 0 : EVT getSetCCResultType(EVT VT) const {
87 0 : return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
88 : }
89 :
90 : /// Pretend all of this node's results are legal.
91 0 : bool IgnoreNodeResults(SDNode *N) const {
92 207833698 : return N->getOpcode() == ISD::TargetConstant ||
93 0 : N->getOpcode() == ISD::Register;
94 : }
95 :
96 : // Bijection from SDValue to unique id. As each created node gets a
97 : // new id we do not need to worry about reuse expunging. Should we
98 : // run out of ids, we can do a one time expensive compactifcation.
99 : typedef unsigned TableId;
100 :
101 : TableId NextValueId = 1;
102 :
103 : SmallDenseMap<SDValue, TableId, 8> ValueToIdMap;
104 : SmallDenseMap<TableId, SDValue, 8> IdToValueMap;
105 :
106 : /// For integer nodes that are below legal width, this map indicates what
107 : /// promoted value to use.
108 : SmallDenseMap<TableId, TableId, 8> PromotedIntegers;
109 :
110 : /// For integer nodes that need to be expanded this map indicates which
111 : /// operands are the expanded version of the input.
112 : SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedIntegers;
113 :
114 : /// For floating-point nodes converted to integers of the same size, this map
115 : /// indicates the converted value to use.
116 : SmallDenseMap<TableId, TableId, 8> SoftenedFloats;
117 :
118 : /// For floating-point nodes that have a smaller precision than the smallest
119 : /// supported precision, this map indicates what promoted value to use.
120 : SmallDenseMap<TableId, TableId, 8> PromotedFloats;
121 :
122 : /// For float nodes that need to be expanded this map indicates which operands
123 : /// are the expanded version of the input.
124 : SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedFloats;
125 :
126 : /// For nodes that are <1 x ty>, this map indicates the scalar value of type
127 : /// 'ty' to use.
128 : SmallDenseMap<TableId, TableId, 8> ScalarizedVectors;
129 :
130 : /// For nodes that need to be split this map indicates which operands are the
131 : /// expanded version of the input.
132 : SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> SplitVectors;
133 :
134 : /// For vector nodes that need to be widened, indicates the widened value to
135 : /// use.
136 : SmallDenseMap<TableId, TableId, 8> WidenedVectors;
137 :
138 : /// For values that have been replaced with another, indicates the replacement
139 : /// value to use.
140 : SmallDenseMap<TableId, TableId, 8> ReplacedValues;
141 :
142 : /// This defines a worklist of nodes to process. In order to be pushed onto
143 : /// this worklist, all operands of a node must have already been processed.
144 : SmallVector<SDNode*, 128> Worklist;
145 :
146 9797419 : TableId getTableId(SDValue V) {
147 : assert(V.getNode() && "Getting TableId on SDValue()");
148 :
149 9797419 : auto I = ValueToIdMap.find(V);
150 9797419 : if (I != ValueToIdMap.end()) {
151 : // replace if there's been a shift.
152 5434876 : RemapId(I->second);
153 : assert(I->second && "All Ids should be nonzero");
154 5434876 : return I->second;
155 : }
156 : // Add if it's not there.
157 4362543 : ValueToIdMap.insert(std::make_pair(V, NextValueId));
158 4362543 : IdToValueMap.insert(std::make_pair(NextValueId, V));
159 4362543 : ++NextValueId;
160 : assert(NextValueId != 0 &&
161 : "Ran out of Ids. Increase id type size or add compactification");
162 4362543 : return NextValueId - 1;
163 : }
164 :
165 : const SDValue &getSDValue(TableId &Id) {
166 6769332 : RemapId(Id);
167 : assert(Id && "TableId should be non-zero");
168 5958755 : return IdToValueMap[Id];
169 : }
170 :
171 : public:
172 1288200 : explicit DAGTypeLegalizer(SelectionDAG &dag)
173 1288200 : : TLI(dag.getTargetLoweringInfo()), DAG(dag),
174 1288200 : ValueTypeActions(TLI.getValueTypeActions()) {
175 : static_assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE,
176 : "Too many value types for ValueTypeActions to hold!");
177 1288201 : }
178 :
179 : /// This is the main entry point for the type legalizer. This does a
180 : /// top-down traversal of the dag, legalizing types as it goes. Returns
181 : /// "true" if it made any changes.
182 : bool run();
183 :
184 445 : void NoteDeletion(SDNode *Old, SDNode *New) {
185 891 : for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
186 446 : TableId NewId = getTableId(SDValue(New, i));
187 446 : TableId OldId = getTableId(SDValue(Old, i));
188 :
189 446 : if (OldId != NewId)
190 446 : ReplacedValues[OldId] = NewId;
191 :
192 : // Delete Node from tables.
193 892 : ValueToIdMap.erase(SDValue(Old, i));
194 446 : IdToValueMap.erase(OldId);
195 446 : PromotedIntegers.erase(OldId);
196 446 : ExpandedIntegers.erase(OldId);
197 446 : SoftenedFloats.erase(OldId);
198 446 : PromotedFloats.erase(OldId);
199 446 : ExpandedFloats.erase(OldId);
200 446 : ScalarizedVectors.erase(OldId);
201 446 : SplitVectors.erase(OldId);
202 446 : WidenedVectors.erase(OldId);
203 : }
204 445 : }
205 :
206 0 : SelectionDAG &getDAG() const { return DAG; }
207 :
208 : private:
209 : SDNode *AnalyzeNewNode(SDNode *N);
210 : void AnalyzeNewValue(SDValue &Val);
211 : void PerformExpensiveChecks();
212 : void RemapId(TableId &Id);
213 : void RemapValue(SDValue &V);
214 :
215 : // Common routines.
216 : SDValue BitConvertToInteger(SDValue Op);
217 : SDValue BitConvertVectorToIntegerVector(SDValue Op);
218 : SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
219 : bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
220 : bool CustomWidenLowerNode(SDNode *N, EVT VT);
221 :
222 : /// Replace each result of the given MERGE_VALUES node with the corresponding
223 : /// input operand, except for the result 'ResNo', for which the corresponding
224 : /// input operand is returned.
225 : SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
226 :
227 : SDValue JoinIntegers(SDValue Lo, SDValue Hi);
228 : SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
229 :
230 : std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
231 : SDNode *Node, bool isSigned);
232 : std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
233 :
234 : SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);
235 :
236 : void ReplaceValueWith(SDValue From, SDValue To);
237 : void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
238 : void SplitInteger(SDValue Op, EVT LoVT, EVT HiVT,
239 : SDValue &Lo, SDValue &Hi);
240 :
241 : void AddToWorklist(SDNode *N) {
242 : N->setNodeId(ReadyToProcess);
243 15301527 : Worklist.push_back(N);
244 : }
245 :
246 : //===--------------------------------------------------------------------===//
247 : // Integer Promotion Support: LegalizeIntegerTypes.cpp
248 : //===--------------------------------------------------------------------===//
249 :
250 : /// Given a processed operand Op which was promoted to a larger integer type,
251 : /// this returns the promoted value. The low bits of the promoted value
252 : /// corresponding to the original type are exactly equal to Op.
253 : /// The extra bits contain rubbish, so the promoted value may need to be zero-
254 : /// or sign-extended from the original type before it is usable (the helpers
255 : /// SExtPromotedInteger and ZExtPromotedInteger can do this for you).
256 : /// For example, if Op is an i16 and was promoted to an i32, then this method
257 : /// returns an i32, the lower 16 bits of which coincide with Op, and the upper
258 : /// 16 bits of which contain rubbish.
259 281946 : SDValue GetPromotedInteger(SDValue Op) {
260 281946 : TableId &PromotedId = PromotedIntegers[getTableId(Op)];
261 281946 : SDValue PromotedOp = getSDValue(PromotedId);
262 : assert(PromotedOp.getNode() && "Operand wasn't promoted?");
263 281946 : return PromotedOp;
264 : }
265 : void SetPromotedInteger(SDValue Op, SDValue Result);
266 :
267 : /// Get a promoted operand and sign extend it to the final size.
268 2478 : SDValue SExtPromotedInteger(SDValue Op) {
269 2478 : EVT OldVT = Op.getValueType();
270 : SDLoc dl(Op);
271 2478 : Op = GetPromotedInteger(Op);
272 2478 : return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op,
273 2478 : DAG.getValueType(OldVT));
274 : }
275 :
276 : /// Get a promoted operand and zero extend it to the final size.
277 16196 : SDValue ZExtPromotedInteger(SDValue Op) {
278 16196 : EVT OldVT = Op.getValueType();
279 : SDLoc dl(Op);
280 16196 : Op = GetPromotedInteger(Op);
281 16196 : return DAG.getZeroExtendInReg(Op, dl, OldVT.getScalarType());
282 : }
283 :
284 : // Integer Result Promotion.
285 : void PromoteIntegerResult(SDNode *N, unsigned ResNo);
286 : SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
287 : SDValue PromoteIntRes_AssertSext(SDNode *N);
288 : SDValue PromoteIntRes_AssertZext(SDNode *N);
289 : SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
290 : SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
291 : SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);
292 : SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
293 : SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
294 : SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
295 : SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N);
296 : SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);
297 : SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
298 : SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
299 : SDValue PromoteIntRes_BITCAST(SDNode *N);
300 : SDValue PromoteIntRes_BSWAP(SDNode *N);
301 : SDValue PromoteIntRes_BITREVERSE(SDNode *N);
302 : SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);
303 : SDValue PromoteIntRes_Constant(SDNode *N);
304 : SDValue PromoteIntRes_CTLZ(SDNode *N);
305 : SDValue PromoteIntRes_CTPOP(SDNode *N);
306 : SDValue PromoteIntRes_CTTZ(SDNode *N);
307 : SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);
308 : SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);
309 : SDValue PromoteIntRes_FP_TO_FP16(SDNode *N);
310 : SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
311 : SDValue PromoteIntRes_LOAD(LoadSDNode *N);
312 : SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
313 : SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
314 : SDValue PromoteIntRes_Overflow(SDNode *N);
315 : SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
316 : SDValue PromoteIntRes_SELECT(SDNode *N);
317 : SDValue PromoteIntRes_VSELECT(SDNode *N);
318 : SDValue PromoteIntRes_SELECT_CC(SDNode *N);
319 : SDValue PromoteIntRes_SETCC(SDNode *N);
320 : SDValue PromoteIntRes_SHL(SDNode *N);
321 : SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);
322 : SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);
323 : SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);
324 : SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);
325 : SDValue PromoteIntRes_SRA(SDNode *N);
326 : SDValue PromoteIntRes_SRL(SDNode *N);
327 : SDValue PromoteIntRes_TRUNCATE(SDNode *N);
328 : SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);
329 : SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
330 : SDValue PromoteIntRes_UNDEF(SDNode *N);
331 : SDValue PromoteIntRes_VAARG(SDNode *N);
332 : SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
333 : SDValue PromoteIntRes_SADDSAT(SDNode *N);
334 :
335 : // Integer Operand Promotion.
336 : bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
337 : SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);
338 : SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);
339 : SDValue PromoteIntOp_BITCAST(SDNode *N);
340 : SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);
341 : SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);
342 : SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo);
343 : SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N);
344 : SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);
345 : SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);
346 : SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);
347 : SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
348 : SDValue PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N);
349 : SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
350 : SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
351 : SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
352 : SDValue PromoteIntOp_Shift(SDNode *N);
353 : SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
354 : SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
355 : SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);
356 : SDValue PromoteIntOp_TRUNCATE(SDNode *N);
357 : SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);
358 : SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);
359 : SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
360 : SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
361 : SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
362 : SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
363 : SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
364 :
365 : void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
366 :
367 : //===--------------------------------------------------------------------===//
368 : // Integer Expansion Support: LegalizeIntegerTypes.cpp
369 : //===--------------------------------------------------------------------===//
370 :
371 : /// Given a processed operand Op which was expanded into two integers of half
372 : /// the size, this returns the two halves. The low bits of Op are exactly
373 : /// equal to the bits of Lo; the high bits exactly equal Hi.
374 : /// For example, if Op is an i64 which was expanded into two i32's, then this
375 : /// method returns the two i32's, with Lo being equal to the lower 32 bits of
376 : /// Op, and Hi being equal to the upper 32 bits.
377 : void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
378 : void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
379 :
380 : // Integer Result Expansion.
381 : void ExpandIntegerResult(SDNode *N, unsigned ResNo);
382 : void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
383 : void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);
384 : void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
385 : void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);
386 : void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
387 : void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
388 : void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
389 : void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
390 : void ExpandIntRes_READCYCLECOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi);
391 : void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
392 : void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);
393 : void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi);
394 : void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
395 : void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi);
396 : void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi);
397 : void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi);
398 :
399 : void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi);
400 : void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
401 : void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi);
402 : void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi);
403 : void ExpandIntRes_ADDSUBCARRY (SDNode *N, SDValue &Lo, SDValue &Hi);
404 : void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi);
405 : void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi);
406 : void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi);
407 : void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
408 : void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi);
409 : void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
410 : void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);
411 : void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);
412 :
413 : void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);
414 :
415 : void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
416 : void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
417 : void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);
418 : void ExpandIntRes_SADDSAT (SDNode *N, SDValue &Lo, SDValue &Hi);
419 :
420 : void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
421 :
422 : void ExpandShiftByConstant(SDNode *N, const APInt &Amt,
423 : SDValue &Lo, SDValue &Hi);
424 : bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
425 : bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
426 :
427 : // Integer Operand Expansion.
428 : bool ExpandIntegerOperand(SDNode *N, unsigned OpNo);
429 : SDValue ExpandIntOp_BR_CC(SDNode *N);
430 : SDValue ExpandIntOp_SELECT_CC(SDNode *N);
431 : SDValue ExpandIntOp_SETCC(SDNode *N);
432 : SDValue ExpandIntOp_SETCCCARRY(SDNode *N);
433 : SDValue ExpandIntOp_Shift(SDNode *N);
434 : SDValue ExpandIntOp_SINT_TO_FP(SDNode *N);
435 : SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
436 : SDValue ExpandIntOp_TRUNCATE(SDNode *N);
437 : SDValue ExpandIntOp_UINT_TO_FP(SDNode *N);
438 : SDValue ExpandIntOp_RETURNADDR(SDNode *N);
439 : SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
440 :
441 : void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
442 : ISD::CondCode &CCCode, const SDLoc &dl);
443 :
444 : //===--------------------------------------------------------------------===//
445 : // Float to Integer Conversion Support: LegalizeFloatTypes.cpp
446 : //===--------------------------------------------------------------------===//
447 :
448 : /// Given an operand Op of Float type, returns the integer if the Op is not
449 : /// supported in target HW and converted to the integer.
450 : /// The integer contains exactly the same bits as Op - only the type changed.
451 : /// For example, if Op is an f32 which was softened to an i32, then this
452 : /// method returns an i32, the bits of which coincide with those of Op.
453 : /// If the Op can be efficiently supported in target HW or the operand must
454 : /// stay in a register, the Op is not converted to an integer.
455 : /// In that case, the given op is returned.
456 4536 : SDValue GetSoftenedFloat(SDValue Op) {
457 4536 : TableId Id = getTableId(Op);
458 4536 : auto Iter = SoftenedFloats.find(Id);
459 4536 : if (Iter == SoftenedFloats.end()) {
460 : assert(isSimpleLegalType(Op.getValueType()) &&
461 : "Operand wasn't converted to integer?");
462 969 : return Op;
463 : }
464 3567 : SDValue SoftenedOp = getSDValue(Iter->second);
465 : assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?");
466 3567 : return SoftenedOp;
467 : }
468 : void SetSoftenedFloat(SDValue Op, SDValue Result);
469 :
470 : // Convert Float Results to Integer for Non-HW-supported Operations.
471 : bool SoftenFloatResult(SDNode *N, unsigned ResNo);
472 : SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
473 : SDValue SoftenFloatRes_BITCAST(SDNode *N, unsigned ResNo);
474 : SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);
475 : SDValue SoftenFloatRes_ConstantFP(SDNode *N, unsigned ResNo);
476 : SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo);
477 : SDValue SoftenFloatRes_FABS(SDNode *N, unsigned ResNo);
478 : SDValue SoftenFloatRes_FMINNUM(SDNode *N);
479 : SDValue SoftenFloatRes_FMAXNUM(SDNode *N);
480 : SDValue SoftenFloatRes_FADD(SDNode *N);
481 : SDValue SoftenFloatRes_FCEIL(SDNode *N);
482 : SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N, unsigned ResNo);
483 : SDValue SoftenFloatRes_FCOS(SDNode *N);
484 : SDValue SoftenFloatRes_FDIV(SDNode *N);
485 : SDValue SoftenFloatRes_FEXP(SDNode *N);
486 : SDValue SoftenFloatRes_FEXP2(SDNode *N);
487 : SDValue SoftenFloatRes_FFLOOR(SDNode *N);
488 : SDValue SoftenFloatRes_FLOG(SDNode *N);
489 : SDValue SoftenFloatRes_FLOG2(SDNode *N);
490 : SDValue SoftenFloatRes_FLOG10(SDNode *N);
491 : SDValue SoftenFloatRes_FMA(SDNode *N);
492 : SDValue SoftenFloatRes_FMUL(SDNode *N);
493 : SDValue SoftenFloatRes_FNEARBYINT(SDNode *N);
494 : SDValue SoftenFloatRes_FNEG(SDNode *N, unsigned ResNo);
495 : SDValue SoftenFloatRes_FP_EXTEND(SDNode *N);
496 : SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N);
497 : SDValue SoftenFloatRes_FP_ROUND(SDNode *N);
498 : SDValue SoftenFloatRes_FPOW(SDNode *N);
499 : SDValue SoftenFloatRes_FPOWI(SDNode *N);
500 : SDValue SoftenFloatRes_FREM(SDNode *N);
501 : SDValue SoftenFloatRes_FRINT(SDNode *N);
502 : SDValue SoftenFloatRes_FROUND(SDNode *N);
503 : SDValue SoftenFloatRes_FSIN(SDNode *N);
504 : SDValue SoftenFloatRes_FSQRT(SDNode *N);
505 : SDValue SoftenFloatRes_FSUB(SDNode *N);
506 : SDValue SoftenFloatRes_FTRUNC(SDNode *N);
507 : SDValue SoftenFloatRes_LOAD(SDNode *N, unsigned ResNo);
508 : SDValue SoftenFloatRes_SELECT(SDNode *N, unsigned ResNo);
509 : SDValue SoftenFloatRes_SELECT_CC(SDNode *N, unsigned ResNo);
510 : SDValue SoftenFloatRes_UNDEF(SDNode *N);
511 : SDValue SoftenFloatRes_VAARG(SDNode *N);
512 : SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
513 :
514 : // Return true if we can skip softening the given operand or SDNode because
515 : // either it was soften before by SoftenFloatResult and references to the
516 : // operand were replaced by ReplaceValueWith or it's value type is legal in HW
517 : // registers and the operand can be left unchanged.
518 : bool CanSkipSoftenFloatOperand(SDNode *N, unsigned OpNo);
519 :
520 : // Convert Float Operand to Integer for Non-HW-supported Operations.
521 : bool SoftenFloatOperand(SDNode *N, unsigned OpNo);
522 : SDValue SoftenFloatOp_BITCAST(SDNode *N);
523 : SDValue SoftenFloatOp_COPY_TO_REG(SDNode *N);
524 : SDValue SoftenFloatOp_BR_CC(SDNode *N);
525 : SDValue SoftenFloatOp_FABS(SDNode *N);
526 : SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N);
527 : SDValue SoftenFloatOp_FNEG(SDNode *N);
528 : SDValue SoftenFloatOp_FP_EXTEND(SDNode *N);
529 : SDValue SoftenFloatOp_FP_ROUND(SDNode *N);
530 : SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);
531 : SDValue SoftenFloatOp_SELECT(SDNode *N);
532 : SDValue SoftenFloatOp_SELECT_CC(SDNode *N);
533 : SDValue SoftenFloatOp_SETCC(SDNode *N);
534 : SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);
535 :
536 : //===--------------------------------------------------------------------===//
537 : // Float Expansion Support: LegalizeFloatTypes.cpp
538 : //===--------------------------------------------------------------------===//
539 :
540 : /// Given a processed operand Op which was expanded into two floating-point
541 : /// values of half the size, this returns the two halves.
542 : /// The low bits of Op are exactly equal to the bits of Lo; the high bits
543 : /// exactly equal Hi. For example, if Op is a ppcf128 which was expanded
544 : /// into two f64's, then this method returns the two f64's, with Lo being
545 : /// equal to the lower 64 bits of Op, and Hi to the upper 64 bits.
546 : void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
547 : void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
548 :
549 : // Float Result Expansion.
550 : void ExpandFloatResult(SDNode *N, unsigned ResNo);
551 : void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);
552 : void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);
553 : void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
554 : void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
555 : void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);
556 : void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);
557 : void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);
558 : void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi);
559 : void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
560 : void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi);
561 : void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi);
562 : void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi);
563 : void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi);
564 : void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi);
565 : void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi);
566 : void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi);
567 : void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi);
568 : void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);
569 : void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi);
570 : void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
571 : void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi);
572 : void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi);
573 : void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi);
574 : void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
575 : void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
576 : void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi);
577 : void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi);
578 : void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
579 : void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi);
580 : void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
581 : void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
582 :
583 : // Float Operand Expansion.
584 : bool ExpandFloatOperand(SDNode *N, unsigned OpNo);
585 : SDValue ExpandFloatOp_BR_CC(SDNode *N);
586 : SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N);
587 : SDValue ExpandFloatOp_FP_ROUND(SDNode *N);
588 : SDValue ExpandFloatOp_FP_TO_SINT(SDNode *N);
589 : SDValue ExpandFloatOp_FP_TO_UINT(SDNode *N);
590 : SDValue ExpandFloatOp_SELECT_CC(SDNode *N);
591 : SDValue ExpandFloatOp_SETCC(SDNode *N);
592 : SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo);
593 :
594 : void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
595 : ISD::CondCode &CCCode, const SDLoc &dl);
596 :
597 : //===--------------------------------------------------------------------===//
598 : // Float promotion support: LegalizeFloatTypes.cpp
599 : //===--------------------------------------------------------------------===//
600 :
601 6145 : SDValue GetPromotedFloat(SDValue Op) {
602 6145 : TableId &PromotedId = PromotedFloats[getTableId(Op)];
603 6145 : SDValue PromotedOp = getSDValue(PromotedId);
604 : assert(PromotedOp.getNode() && "Operand wasn't promoted?");
605 6145 : return PromotedOp;
606 : }
607 : void SetPromotedFloat(SDValue Op, SDValue Result);
608 :
609 : void PromoteFloatResult(SDNode *N, unsigned ResNo);
610 : SDValue PromoteFloatRes_BITCAST(SDNode *N);
611 : SDValue PromoteFloatRes_BinOp(SDNode *N);
612 : SDValue PromoteFloatRes_ConstantFP(SDNode *N);
613 : SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);
614 : SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);
615 : SDValue PromoteFloatRes_FMAD(SDNode *N);
616 : SDValue PromoteFloatRes_FPOWI(SDNode *N);
617 : SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
618 : SDValue PromoteFloatRes_LOAD(SDNode *N);
619 : SDValue PromoteFloatRes_SELECT(SDNode *N);
620 : SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
621 : SDValue PromoteFloatRes_UnaryOp(SDNode *N);
622 : SDValue PromoteFloatRes_UNDEF(SDNode *N);
623 : SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
624 :
625 : bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
626 : SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
627 : SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
628 : SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
629 : SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
630 : SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
631 : SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
632 : SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
633 :
634 : //===--------------------------------------------------------------------===//
635 : // Scalarization Support: LegalizeVectorTypes.cpp
636 : //===--------------------------------------------------------------------===//
637 :
638 : /// Given a processed one-element vector Op which was scalarized to its
639 : /// element type, this returns the element. For example, if Op is a v1i32,
640 : /// Op = < i32 val >, this method returns val, an i32.
641 63730 : SDValue GetScalarizedVector(SDValue Op) {
642 63730 : TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)];
643 63730 : SDValue ScalarizedOp = getSDValue(ScalarizedId);
644 : assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?");
645 63730 : return ScalarizedOp;
646 : }
647 : void SetScalarizedVector(SDValue Op, SDValue Result);
648 :
649 : // Vector Result Scalarization: <1 x ty> -> ty.
650 : void ScalarizeVectorResult(SDNode *N, unsigned ResNo);
651 : SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
652 : SDValue ScalarizeVecRes_BinOp(SDNode *N);
653 : SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
654 : SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
655 : SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
656 : SDValue ScalarizeVecRes_InregOp(SDNode *N);
657 : SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
658 :
659 : SDValue ScalarizeVecRes_BITCAST(SDNode *N);
660 : SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
661 : SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
662 : SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
663 : SDValue ScalarizeVecRes_FPOWI(SDNode *N);
664 : SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);
665 : SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);
666 : SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
667 : SDValue ScalarizeVecRes_VSELECT(SDNode *N);
668 : SDValue ScalarizeVecRes_SELECT(SDNode *N);
669 : SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
670 : SDValue ScalarizeVecRes_SETCC(SDNode *N);
671 : SDValue ScalarizeVecRes_UNDEF(SDNode *N);
672 : SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);
673 :
674 : // Vector Operand Scalarization: <1 x ty> -> ty.
675 : bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);
676 : SDValue ScalarizeVecOp_BITCAST(SDNode *N);
677 : SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
678 : SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
679 : SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
680 : SDValue ScalarizeVecOp_VSELECT(SDNode *N);
681 : SDValue ScalarizeVecOp_VSETCC(SDNode *N);
682 : SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);
683 : SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);
684 :
685 : //===--------------------------------------------------------------------===//
686 : // Vector Splitting Support: LegalizeVectorTypes.cpp
687 : //===--------------------------------------------------------------------===//
688 :
689 : /// Given a processed vector Op which was split into vectors of half the size,
690 : /// this method returns the halves. The first elements of Op coincide with the
691 : /// elements of Lo; the remaining elements of Op coincide with the elements of
692 : /// Hi: Op is what you would get by concatenating Lo and Hi.
693 : /// For example, if Op is a v8i32 that was split into two v4i32's, then this
694 : /// method returns the two v4i32's, with Lo corresponding to the first 4
695 : /// elements of Op, and Hi to the last 4 elements.
696 : void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
697 : void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
698 :
699 : // Vector Result Splitting: <128 x ty> -> 2 x <64 x ty>.
700 : void SplitVectorResult(SDNode *N, unsigned ResNo);
701 : void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
702 : void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
703 : void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
704 : void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
705 : void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
706 : void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
707 : void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
708 :
709 : void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
710 : void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
711 : void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
712 : void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
713 : void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
714 : void SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi);
715 : void SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, SDValue &Hi);
716 : void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
717 : void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
718 : void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
719 : void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
720 : void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
721 : void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
722 : void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
723 : SDValue &Hi);
724 :
725 : // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>.
726 : bool SplitVectorOperand(SDNode *N, unsigned OpNo);
727 : SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
728 : SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
729 : SDValue SplitVecOp_UnaryOp(SDNode *N);
730 : SDValue SplitVecOp_TruncateHelper(SDNode *N);
731 :
732 : SDValue SplitVecOp_BITCAST(SDNode *N);
733 : SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
734 : SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
735 : SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);
736 : SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
737 : SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
738 : SDValue SplitVecOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
739 : SDValue SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, unsigned OpNo);
740 : SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
741 : SDValue SplitVecOp_VSETCC(SDNode *N);
742 : SDValue SplitVecOp_FP_ROUND(SDNode *N);
743 : SDValue SplitVecOp_FCOPYSIGN(SDNode *N);
744 :
745 : //===--------------------------------------------------------------------===//
746 : // Vector Widening Support: LegalizeVectorTypes.cpp
747 : //===--------------------------------------------------------------------===//
748 :
749 : /// Given a processed vector Op which was widened into a larger vector, this
750 : /// method returns the larger vector. The elements of the returned vector
751 : /// consist of the elements of Op followed by elements containing rubbish.
752 : /// For example, if Op is a v2i32 that was widened to a v4i32, then this
753 : /// method returns a v4i32 for which the first two elements are the same as
754 : /// those of Op, while the last two elements contain rubbish.
755 16943 : SDValue GetWidenedVector(SDValue Op) {
756 16943 : TableId &WidenedId = WidenedVectors[getTableId(Op)];
757 16943 : SDValue WidenedOp = getSDValue(WidenedId);
758 : assert(WidenedOp.getNode() && "Operand wasn't widened?");
759 16943 : return WidenedOp;
760 : }
761 : void SetWidenedVector(SDValue Op, SDValue Result);
762 :
763 : // Widen Vector Result Promotion.
764 : void WidenVectorResult(SDNode *N, unsigned ResNo);
765 : SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
766 : SDValue WidenVecRes_BITCAST(SDNode* N);
767 : SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
768 : SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
769 : SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);
770 : SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
771 : SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
772 : SDValue WidenVecRes_LOAD(SDNode* N);
773 : SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
774 : SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
775 : SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N);
776 : SDValue WidenVecRes_SELECT(SDNode* N);
777 : SDValue WidenVSELECTAndMask(SDNode *N);
778 : SDValue WidenVecRes_SELECT_CC(SDNode* N);
779 : SDValue WidenVecRes_SETCC(SDNode* N);
780 : SDValue WidenVecRes_UNDEF(SDNode *N);
781 : SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
782 :
783 : SDValue WidenVecRes_Ternary(SDNode *N);
784 : SDValue WidenVecRes_Binary(SDNode *N);
785 : SDValue WidenVecRes_BinaryCanTrap(SDNode *N);
786 : SDValue WidenVecRes_StrictFP(SDNode *N);
787 : SDValue WidenVecRes_Convert(SDNode *N);
788 : SDValue WidenVecRes_FCOPYSIGN(SDNode *N);
789 : SDValue WidenVecRes_POWI(SDNode *N);
790 : SDValue WidenVecRes_Shift(SDNode *N);
791 : SDValue WidenVecRes_Unary(SDNode *N);
792 : SDValue WidenVecRes_InregOp(SDNode *N);
793 :
794 : // Widen Vector Operand.
795 : bool WidenVectorOperand(SDNode *N, unsigned OpNo);
796 : SDValue WidenVecOp_BITCAST(SDNode *N);
797 : SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
798 : SDValue WidenVecOp_EXTEND(SDNode *N);
799 : SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
800 : SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
801 : SDValue WidenVecOp_STORE(SDNode* N);
802 : SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
803 : SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
804 : SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
805 : SDValue WidenVecOp_SETCC(SDNode* N);
806 :
807 : SDValue WidenVecOp_Convert(SDNode *N);
808 : SDValue WidenVecOp_FCOPYSIGN(SDNode *N);
809 :
810 : //===--------------------------------------------------------------------===//
811 : // Vector Widening Utilities Support: LegalizeVectorTypes.cpp
812 : //===--------------------------------------------------------------------===//
813 :
814 : /// Helper function to generate a set of loads to load a vector with a
815 : /// resulting wider type. It takes:
816 : /// LdChain: list of chains for the load to be generated.
817 : /// Ld: load to widen
818 : SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
819 : LoadSDNode *LD);
820 :
821 : /// Helper function to generate a set of extension loads to load a vector with
822 : /// a resulting wider type. It takes:
823 : /// LdChain: list of chains for the load to be generated.
824 : /// Ld: load to widen
825 : /// ExtType: extension element type
826 : SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
827 : LoadSDNode *LD, ISD::LoadExtType ExtType);
828 :
829 : /// Helper function to generate a set of stores to store a widen vector into
830 : /// non-widen memory.
831 : /// StChain: list of chains for the stores we have generated
832 : /// ST: store of a widen value
833 : void GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST);
834 :
835 : /// Helper function to generate a set of stores to store a truncate widen
836 : /// vector into non-widen memory.
837 : /// StChain: list of chains for the stores we have generated
838 : /// ST: store of a widen value
839 : void GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
840 : StoreSDNode *ST);
841 :
842 : /// Modifies a vector input (widen or narrows) to a vector of NVT. The
843 : /// input vector must have the same element type as NVT.
844 : /// When FillWithZeroes is "on" the vector will be widened with zeroes.
845 : /// By default, the vector will be widened with undefined values.
846 : SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);
847 :
848 : /// Return a mask of vector type MaskVT to replace InMask. Also adjust
849 : /// MaskVT to ToMaskVT if needed with vector extension or truncation.
850 : SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);
851 :
852 : /// Get the target mask VT, and widen if needed.
853 : EVT getSETCCWidenedResultTy(SDValue SetCC);
854 :
855 : //===--------------------------------------------------------------------===//
856 : // Generic Splitting: LegalizeTypesGeneric.cpp
857 : //===--------------------------------------------------------------------===//
858 :
859 : // Legalization methods which only use that the illegal type is split into two
860 : // not necessarily identical types. As such they can be used for splitting
861 : // vectors and expanding integers and floats.
862 :
863 4949 : void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
864 9898 : if (Op.getValueType().isVector())
865 2550 : GetSplitVector(Op, Lo, Hi);
866 2399 : else if (Op.getValueType().isInteger())
867 2385 : GetExpandedInteger(Op, Lo, Hi);
868 : else
869 14 : GetExpandedFloat(Op, Lo, Hi);
870 4949 : }
871 :
872 : /// Use ISD::EXTRACT_ELEMENT nodes to extract the low and high parts of the
873 : /// given value.
874 : void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
875 :
876 : // Generic Result Splitting.
877 : void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
878 : SDValue &Lo, SDValue &Hi);
879 : void SplitRes_SELECT (SDNode *N, SDValue &Lo, SDValue &Hi);
880 : void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
881 : void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
882 :
883 : //===--------------------------------------------------------------------===//
884 : // Generic Expansion: LegalizeTypesGeneric.cpp
885 : //===--------------------------------------------------------------------===//
886 :
887 : // Legalization methods which only use that the illegal type is split into two
888 : // identical types of half the size, and that the Lo/Hi part is stored first
889 : // in memory on little/big-endian machines, followed by the Hi/Lo part. As
890 : // such they can be used for expanding integers and floats.
891 :
892 187166 : void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
893 374332 : if (Op.getValueType().isInteger())
894 186951 : GetExpandedInteger(Op, Lo, Hi);
895 : else
896 215 : GetExpandedFloat(Op, Lo, Hi);
897 187166 : }
898 :
899 :
900 : /// This function will split the integer \p Op into \p NumElements
901 : /// operations of type \p EltVT and store them in \p Ops.
902 : void IntegerToVector(SDValue Op, unsigned NumElements,
903 : SmallVectorImpl<SDValue> &Ops, EVT EltVT);
904 :
905 : // Generic Result Expansion.
906 : void ExpandRes_MERGE_VALUES (SDNode *N, unsigned ResNo,
907 : SDValue &Lo, SDValue &Hi);
908 : void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi);
909 : void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi);
910 : void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi);
911 : void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
912 : void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi);
913 : void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi);
914 :
915 : // Generic Operand Expansion.
916 : SDValue ExpandOp_BITCAST (SDNode *N);
917 : SDValue ExpandOp_BUILD_VECTOR (SDNode *N);
918 : SDValue ExpandOp_EXTRACT_ELEMENT (SDNode *N);
919 : SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N);
920 : SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N);
921 : SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo);
922 : };
923 :
924 : } // end namespace llvm.
925 :
926 : #endif
|
