LLVM API Documentation
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 00002 // 00003 // The LLVM Compiler Infrastructure 00004 // 00005 // This file is distributed under the University of Illinois Open Source 00006 // License. See LICENSE.TXT for details. 00007 // 00008 //===----------------------------------------------------------------------===// 00009 // 00010 // This implements the SelectionDAG class. 00011 // 00012 //===----------------------------------------------------------------------===// 00013 00014 #include "llvm/CodeGen/SelectionDAG.h" 00015 #include "SDNodeDbgValue.h" 00016 #include "llvm/ADT/SetVector.h" 00017 #include "llvm/ADT/SmallPtrSet.h" 00018 #include "llvm/ADT/SmallSet.h" 00019 #include "llvm/ADT/SmallVector.h" 00020 #include "llvm/ADT/StringExtras.h" 00021 #include "llvm/Analysis/TargetTransformInfo.h" 00022 #include "llvm/Analysis/ValueTracking.h" 00023 #include "llvm/Assembly/Writer.h" 00024 #include "llvm/CodeGen/MachineBasicBlock.h" 00025 #include "llvm/CodeGen/MachineConstantPool.h" 00026 #include "llvm/CodeGen/MachineFrameInfo.h" 00027 #include "llvm/CodeGen/MachineModuleInfo.h" 00028 #include "llvm/DebugInfo.h" 00029 #include "llvm/IR/CallingConv.h" 00030 #include "llvm/IR/Constants.h" 00031 #include "llvm/IR/DataLayout.h" 00032 #include "llvm/IR/DerivedTypes.h" 00033 #include "llvm/IR/Function.h" 00034 #include "llvm/IR/GlobalAlias.h" 00035 #include "llvm/IR/GlobalVariable.h" 00036 #include "llvm/IR/Intrinsics.h" 00037 #include "llvm/Support/CommandLine.h" 00038 #include "llvm/Support/Debug.h" 00039 #include "llvm/Support/ErrorHandling.h" 00040 #include "llvm/Support/ManagedStatic.h" 00041 #include "llvm/Support/MathExtras.h" 00042 #include "llvm/Support/Mutex.h" 00043 #include "llvm/Support/raw_ostream.h" 00044 #include "llvm/Target/TargetInstrInfo.h" 00045 #include "llvm/Target/TargetIntrinsicInfo.h" 00046 #include "llvm/Target/TargetLowering.h" 00047 #include "llvm/Target/TargetMachine.h" 00048 #include "llvm/Target/TargetOptions.h" 00049 #include "llvm/Target/TargetRegisterInfo.h" 00050 #include "llvm/Target/TargetSelectionDAGInfo.h" 00051 #include <algorithm> 00052 #include <cmath> 00053 using namespace llvm; 00054 00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the 00056 /// specified members. 00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 00058 SDVTList Res = {VTs, NumVTs}; 00059 return Res; 00060 } 00061 00062 // Default null implementations of the callbacks. 00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 00065 00066 //===----------------------------------------------------------------------===// 00067 // ConstantFPSDNode Class 00068 //===----------------------------------------------------------------------===// 00069 00070 /// isExactlyValue - We don't rely on operator== working on double values, as 00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0. 00072 /// As such, this method can be used to do an exact bit-for-bit comparison of 00073 /// two floating point values. 00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 00075 return getValueAPF().bitwiseIsEqual(V); 00076 } 00077 00078 bool ConstantFPSDNode::isValueValidForType(EVT VT, 00079 const APFloat& Val) { 00080 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 00081 00082 // convert modifies in place, so make a copy. 00083 APFloat Val2 = APFloat(Val); 00084 bool losesInfo; 00085 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 00086 APFloat::rmNearestTiesToEven, 00087 &losesInfo); 00088 return !losesInfo; 00089 } 00090 00091 //===----------------------------------------------------------------------===// 00092 // ISD Namespace 00093 //===----------------------------------------------------------------------===// 00094 00095 /// isBuildVectorAllOnes - Return true if the specified node is a 00096 /// BUILD_VECTOR where all of the elements are ~0 or undef. 00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) { 00098 // Look through a bit convert. 00099 if (N->getOpcode() == ISD::BITCAST) 00100 N = N->getOperand(0).getNode(); 00101 00102 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 00103 00104 unsigned i = 0, e = N->getNumOperands(); 00105 00106 // Skip over all of the undef values. 00107 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 00108 ++i; 00109 00110 // Do not accept an all-undef vector. 00111 if (i == e) return false; 00112 00113 // Do not accept build_vectors that aren't all constants or which have non-~0 00114 // elements. We have to be a bit careful here, as the type of the constant 00115 // may not be the same as the type of the vector elements due to type 00116 // legalization (the elements are promoted to a legal type for the target and 00117 // a vector of a type may be legal when the base element type is not). 00118 // We only want to check enough bits to cover the vector elements, because 00119 // we care if the resultant vector is all ones, not whether the individual 00120 // constants are. 00121 SDValue NotZero = N->getOperand(i); 00122 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 00123 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 00124 if (CN->getAPIntValue().countTrailingOnes() < EltSize) 00125 return false; 00126 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 00127 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 00128 return false; 00129 } else 00130 return false; 00131 00132 // Okay, we have at least one ~0 value, check to see if the rest match or are 00133 // undefs. Even with the above element type twiddling, this should be OK, as 00134 // the same type legalization should have applied to all the elements. 00135 for (++i; i != e; ++i) 00136 if (N->getOperand(i) != NotZero && 00137 N->getOperand(i).getOpcode() != ISD::UNDEF) 00138 return false; 00139 return true; 00140 } 00141 00142 00143 /// isBuildVectorAllZeros - Return true if the specified node is a 00144 /// BUILD_VECTOR where all of the elements are 0 or undef. 00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) { 00146 // Look through a bit convert. 00147 if (N->getOpcode() == ISD::BITCAST) 00148 N = N->getOperand(0).getNode(); 00149 00150 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 00151 00152 unsigned i = 0, e = N->getNumOperands(); 00153 00154 // Skip over all of the undef values. 00155 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 00156 ++i; 00157 00158 // Do not accept an all-undef vector. 00159 if (i == e) return false; 00160 00161 // Do not accept build_vectors that aren't all constants or which have non-0 00162 // elements. 00163 SDValue Zero = N->getOperand(i); 00164 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) { 00165 if (!CN->isNullValue()) 00166 return false; 00167 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) { 00168 if (!CFPN->getValueAPF().isPosZero()) 00169 return false; 00170 } else 00171 return false; 00172 00173 // Okay, we have at least one 0 value, check to see if the rest match or are 00174 // undefs. 00175 for (++i; i != e; ++i) 00176 if (N->getOperand(i) != Zero && 00177 N->getOperand(i).getOpcode() != ISD::UNDEF) 00178 return false; 00179 return true; 00180 } 00181 00182 /// isScalarToVector - Return true if the specified node is a 00183 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 00184 /// element is not an undef. 00185 bool ISD::isScalarToVector(const SDNode *N) { 00186 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 00187 return true; 00188 00189 if (N->getOpcode() != ISD::BUILD_VECTOR) 00190 return false; 00191 if (N->getOperand(0).getOpcode() == ISD::UNDEF) 00192 return false; 00193 unsigned NumElems = N->getNumOperands(); 00194 if (NumElems == 1) 00195 return false; 00196 for (unsigned i = 1; i < NumElems; ++i) { 00197 SDValue V = N->getOperand(i); 00198 if (V.getOpcode() != ISD::UNDEF) 00199 return false; 00200 } 00201 return true; 00202 } 00203 00204 /// allOperandsUndef - Return true if the node has at least one operand 00205 /// and all operands of the specified node are ISD::UNDEF. 00206 bool ISD::allOperandsUndef(const SDNode *N) { 00207 // Return false if the node has no operands. 00208 // This is "logically inconsistent" with the definition of "all" but 00209 // is probably the desired behavior. 00210 if (N->getNumOperands() == 0) 00211 return false; 00212 00213 for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i) 00214 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 00215 return false; 00216 00217 return true; 00218 } 00219 00220 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 00221 /// when given the operation for (X op Y). 00222 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 00223 // To perform this operation, we just need to swap the L and G bits of the 00224 // operation. 00225 unsigned OldL = (Operation >> 2) & 1; 00226 unsigned OldG = (Operation >> 1) & 1; 00227 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 00228 (OldL << 1) | // New G bit 00229 (OldG << 2)); // New L bit. 00230 } 00231 00232 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where 00233 /// 'op' is a valid SetCC operation. 00234 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 00235 unsigned Operation = Op; 00236 if (isInteger) 00237 Operation ^= 7; // Flip L, G, E bits, but not U. 00238 else 00239 Operation ^= 15; // Flip all of the condition bits. 00240 00241 if (Operation > ISD::SETTRUE2) 00242 Operation &= ~8; // Don't let N and U bits get set. 00243 00244 return ISD::CondCode(Operation); 00245 } 00246 00247 00248 /// isSignedOp - For an integer comparison, return 1 if the comparison is a 00249 /// signed operation and 2 if the result is an unsigned comparison. Return zero 00250 /// if the operation does not depend on the sign of the input (setne and seteq). 00251 static int isSignedOp(ISD::CondCode Opcode) { 00252 switch (Opcode) { 00253 default: llvm_unreachable("Illegal integer setcc operation!"); 00254 case ISD::SETEQ: 00255 case ISD::SETNE: return 0; 00256 case ISD::SETLT: 00257 case ISD::SETLE: 00258 case ISD::SETGT: 00259 case ISD::SETGE: return 1; 00260 case ISD::SETULT: 00261 case ISD::SETULE: 00262 case ISD::SETUGT: 00263 case ISD::SETUGE: return 2; 00264 } 00265 } 00266 00267 /// getSetCCOrOperation - Return the result of a logical OR between different 00268 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 00269 /// returns SETCC_INVALID if it is not possible to represent the resultant 00270 /// comparison. 00271 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 00272 bool isInteger) { 00273 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 00274 // Cannot fold a signed integer setcc with an unsigned integer setcc. 00275 return ISD::SETCC_INVALID; 00276 00277 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 00278 00279 // If the N and U bits get set then the resultant comparison DOES suddenly 00280 // care about orderedness, and is true when ordered. 00281 if (Op > ISD::SETTRUE2) 00282 Op &= ~16; // Clear the U bit if the N bit is set. 00283 00284 // Canonicalize illegal integer setcc's. 00285 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 00286 Op = ISD::SETNE; 00287 00288 return ISD::CondCode(Op); 00289 } 00290 00291 /// getSetCCAndOperation - Return the result of a logical AND between different 00292 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 00293 /// function returns zero if it is not possible to represent the resultant 00294 /// comparison. 00295 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 00296 bool isInteger) { 00297 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 00298 // Cannot fold a signed setcc with an unsigned setcc. 00299 return ISD::SETCC_INVALID; 00300 00301 // Combine all of the condition bits. 00302 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 00303 00304 // Canonicalize illegal integer setcc's. 00305 if (isInteger) { 00306 switch (Result) { 00307 default: break; 00308 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 00309 case ISD::SETOEQ: // SETEQ & SETU[LG]E 00310 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 00311 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 00312 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 00313 } 00314 } 00315 00316 return Result; 00317 } 00318 00319 //===----------------------------------------------------------------------===// 00320 // SDNode Profile Support 00321 //===----------------------------------------------------------------------===// 00322 00323 /// AddNodeIDOpcode - Add the node opcode to the NodeID data. 00324 /// 00325 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 00326 ID.AddInteger(OpC); 00327 } 00328 00329 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 00330 /// solely with their pointer. 00331 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 00332 ID.AddPointer(VTList.VTs); 00333 } 00334 00335 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 00336 /// 00337 static void AddNodeIDOperands(FoldingSetNodeID &ID, 00338 const SDValue *Ops, unsigned NumOps) { 00339 for (; NumOps; --NumOps, ++Ops) { 00340 ID.AddPointer(Ops->getNode()); 00341 ID.AddInteger(Ops->getResNo()); 00342 } 00343 } 00344 00345 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 00346 /// 00347 static void AddNodeIDOperands(FoldingSetNodeID &ID, 00348 const SDUse *Ops, unsigned NumOps) { 00349 for (; NumOps; --NumOps, ++Ops) { 00350 ID.AddPointer(Ops->getNode()); 00351 ID.AddInteger(Ops->getResNo()); 00352 } 00353 } 00354 00355 static void AddNodeIDNode(FoldingSetNodeID &ID, 00356 unsigned short OpC, SDVTList VTList, 00357 const SDValue *OpList, unsigned N) { 00358 AddNodeIDOpcode(ID, OpC); 00359 AddNodeIDValueTypes(ID, VTList); 00360 AddNodeIDOperands(ID, OpList, N); 00361 } 00362 00363 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to 00364 /// the NodeID data. 00365 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 00366 switch (N->getOpcode()) { 00367 case ISD::TargetExternalSymbol: 00368 case ISD::ExternalSymbol: 00369 llvm_unreachable("Should only be used on nodes with operands"); 00370 default: break; // Normal nodes don't need extra info. 00371 case ISD::TargetConstant: 00372 case ISD::Constant: 00373 ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 00374 break; 00375 case ISD::TargetConstantFP: 00376 case ISD::ConstantFP: { 00377 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 00378 break; 00379 } 00380 case ISD::TargetGlobalAddress: 00381 case ISD::GlobalAddress: 00382 case ISD::TargetGlobalTLSAddress: 00383 case ISD::GlobalTLSAddress: { 00384 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 00385 ID.AddPointer(GA->getGlobal()); 00386 ID.AddInteger(GA->getOffset()); 00387 ID.AddInteger(GA->getTargetFlags()); 00388 ID.AddInteger(GA->getAddressSpace()); 00389 break; 00390 } 00391 case ISD::BasicBlock: 00392 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 00393 break; 00394 case ISD::Register: 00395 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 00396 break; 00397 case ISD::RegisterMask: 00398 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 00399 break; 00400 case ISD::SRCVALUE: 00401 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 00402 break; 00403 case ISD::FrameIndex: 00404 case ISD::TargetFrameIndex: 00405 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 00406 break; 00407 case ISD::JumpTable: 00408 case ISD::TargetJumpTable: 00409 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 00410 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 00411 break; 00412 case ISD::ConstantPool: 00413 case ISD::TargetConstantPool: { 00414 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 00415 ID.AddInteger(CP->getAlignment()); 00416 ID.AddInteger(CP->getOffset()); 00417 if (CP->isMachineConstantPoolEntry()) 00418 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 00419 else 00420 ID.AddPointer(CP->getConstVal()); 00421 ID.AddInteger(CP->getTargetFlags()); 00422 break; 00423 } 00424 case ISD::TargetIndex: { 00425 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 00426 ID.AddInteger(TI->getIndex()); 00427 ID.AddInteger(TI->getOffset()); 00428 ID.AddInteger(TI->getTargetFlags()); 00429 break; 00430 } 00431 case ISD::LOAD: { 00432 const LoadSDNode *LD = cast<LoadSDNode>(N); 00433 ID.AddInteger(LD->getMemoryVT().getRawBits()); 00434 ID.AddInteger(LD->getRawSubclassData()); 00435 ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 00436 break; 00437 } 00438 case ISD::STORE: { 00439 const StoreSDNode *ST = cast<StoreSDNode>(N); 00440 ID.AddInteger(ST->getMemoryVT().getRawBits()); 00441 ID.AddInteger(ST->getRawSubclassData()); 00442 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 00443 break; 00444 } 00445 case ISD::ATOMIC_CMP_SWAP: 00446 case ISD::ATOMIC_SWAP: 00447 case ISD::ATOMIC_LOAD_ADD: 00448 case ISD::ATOMIC_LOAD_SUB: 00449 case ISD::ATOMIC_LOAD_AND: 00450 case ISD::ATOMIC_LOAD_OR: 00451 case ISD::ATOMIC_LOAD_XOR: 00452 case ISD::ATOMIC_LOAD_NAND: 00453 case ISD::ATOMIC_LOAD_MIN: 00454 case ISD::ATOMIC_LOAD_MAX: 00455 case ISD::ATOMIC_LOAD_UMIN: 00456 case ISD::ATOMIC_LOAD_UMAX: 00457 case ISD::ATOMIC_LOAD: 00458 case ISD::ATOMIC_STORE: { 00459 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 00460 ID.AddInteger(AT->getMemoryVT().getRawBits()); 00461 ID.AddInteger(AT->getRawSubclassData()); 00462 ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 00463 break; 00464 } 00465 case ISD::PREFETCH: { 00466 const MemSDNode *PF = cast<MemSDNode>(N); 00467 ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 00468 break; 00469 } 00470 case ISD::VECTOR_SHUFFLE: { 00471 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 00472 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 00473 i != e; ++i) 00474 ID.AddInteger(SVN->getMaskElt(i)); 00475 break; 00476 } 00477 case ISD::TargetBlockAddress: 00478 case ISD::BlockAddress: { 00479 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 00480 ID.AddPointer(BA->getBlockAddress()); 00481 ID.AddInteger(BA->getOffset()); 00482 ID.AddInteger(BA->getTargetFlags()); 00483 break; 00484 } 00485 } // end switch (N->getOpcode()) 00486 00487 // Target specific memory nodes could also have address spaces to check. 00488 if (N->isTargetMemoryOpcode()) 00489 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 00490 } 00491 00492 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 00493 /// data. 00494 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 00495 AddNodeIDOpcode(ID, N->getOpcode()); 00496 // Add the return value info. 00497 AddNodeIDValueTypes(ID, N->getVTList()); 00498 // Add the operand info. 00499 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 00500 00501 // Handle SDNode leafs with special info. 00502 AddNodeIDCustom(ID, N); 00503 } 00504 00505 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in 00506 /// the CSE map that carries volatility, temporalness, indexing mode, and 00507 /// extension/truncation information. 00508 /// 00509 static inline unsigned 00510 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 00511 bool isNonTemporal, bool isInvariant) { 00512 assert((ConvType & 3) == ConvType && 00513 "ConvType may not require more than 2 bits!"); 00514 assert((AM & 7) == AM && 00515 "AM may not require more than 3 bits!"); 00516 return ConvType | 00517 (AM << 2) | 00518 (isVolatile << 5) | 00519 (isNonTemporal << 6) | 00520 (isInvariant << 7); 00521 } 00522 00523 //===----------------------------------------------------------------------===// 00524 // SelectionDAG Class 00525 //===----------------------------------------------------------------------===// 00526 00527 /// doNotCSE - Return true if CSE should not be performed for this node. 00528 static bool doNotCSE(SDNode *N) { 00529 if (N->getValueType(0) == MVT::Glue) 00530 return true; // Never CSE anything that produces a flag. 00531 00532 switch (N->getOpcode()) { 00533 default: break; 00534 case ISD::HANDLENODE: 00535 case ISD::EH_LABEL: 00536 return true; // Never CSE these nodes. 00537 } 00538 00539 // Check that remaining values produced are not flags. 00540 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 00541 if (N->getValueType(i) == MVT::Glue) 00542 return true; // Never CSE anything that produces a flag. 00543 00544 return false; 00545 } 00546 00547 /// RemoveDeadNodes - This method deletes all unreachable nodes in the 00548 /// SelectionDAG. 00549 void SelectionDAG::RemoveDeadNodes() { 00550 // Create a dummy node (which is not added to allnodes), that adds a reference 00551 // to the root node, preventing it from being deleted. 00552 HandleSDNode Dummy(getRoot()); 00553 00554 SmallVector<SDNode*, 128> DeadNodes; 00555 00556 // Add all obviously-dead nodes to the DeadNodes worklist. 00557 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 00558 if (I->use_empty()) 00559 DeadNodes.push_back(I); 00560 00561 RemoveDeadNodes(DeadNodes); 00562 00563 // If the root changed (e.g. it was a dead load, update the root). 00564 setRoot(Dummy.getValue()); 00565 } 00566 00567 /// RemoveDeadNodes - This method deletes the unreachable nodes in the 00568 /// given list, and any nodes that become unreachable as a result. 00569 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 00570 00571 // Process the worklist, deleting the nodes and adding their uses to the 00572 // worklist. 00573 while (!DeadNodes.empty()) { 00574 SDNode *N = DeadNodes.pop_back_val(); 00575 00576 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 00577 DUL->NodeDeleted(N, 0); 00578 00579 // Take the node out of the appropriate CSE map. 00580 RemoveNodeFromCSEMaps(N); 00581 00582 // Next, brutally remove the operand list. This is safe to do, as there are 00583 // no cycles in the graph. 00584 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 00585 SDUse &Use = *I++; 00586 SDNode *Operand = Use.getNode(); 00587 Use.set(SDValue()); 00588 00589 // Now that we removed this operand, see if there are no uses of it left. 00590 if (Operand->use_empty()) 00591 DeadNodes.push_back(Operand); 00592 } 00593 00594 DeallocateNode(N); 00595 } 00596 } 00597 00598 void SelectionDAG::RemoveDeadNode(SDNode *N){ 00599 SmallVector<SDNode*, 16> DeadNodes(1, N); 00600 00601 // Create a dummy node that adds a reference to the root node, preventing 00602 // it from being deleted. (This matters if the root is an operand of the 00603 // dead node.) 00604 HandleSDNode Dummy(getRoot()); 00605 00606 RemoveDeadNodes(DeadNodes); 00607 } 00608 00609 void SelectionDAG::DeleteNode(SDNode *N) { 00610 // First take this out of the appropriate CSE map. 00611 RemoveNodeFromCSEMaps(N); 00612 00613 // Finally, remove uses due to operands of this node, remove from the 00614 // AllNodes list, and delete the node. 00615 DeleteNodeNotInCSEMaps(N); 00616 } 00617 00618 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 00619 assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 00620 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 00621 00622 // Drop all of the operands and decrement used node's use counts. 00623 N->DropOperands(); 00624 00625 DeallocateNode(N); 00626 } 00627 00628 void SelectionDAG::DeallocateNode(SDNode *N) { 00629 if (N->OperandsNeedDelete) 00630 delete[] N->OperandList; 00631 00632 // Set the opcode to DELETED_NODE to help catch bugs when node 00633 // memory is reallocated. 00634 N->NodeType = ISD::DELETED_NODE; 00635 00636 NodeAllocator.Deallocate(AllNodes.remove(N)); 00637 00638 // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 00639 ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 00640 for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 00641 DbgVals[i]->setIsInvalidated(); 00642 } 00643 00644 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 00645 /// correspond to it. This is useful when we're about to delete or repurpose 00646 /// the node. We don't want future request for structurally identical nodes 00647 /// to return N anymore. 00648 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 00649 bool Erased = false; 00650 switch (N->getOpcode()) { 00651 case ISD::HANDLENODE: return false; // noop. 00652 case ISD::CONDCODE: 00653 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 00654 "Cond code doesn't exist!"); 00655 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 00656 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 00657 break; 00658 case ISD::ExternalSymbol: 00659 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 00660 break; 00661 case ISD::TargetExternalSymbol: { 00662 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 00663 Erased = TargetExternalSymbols.erase( 00664 std::pair<std::string,unsigned char>(ESN->getSymbol(), 00665 ESN->getTargetFlags())); 00666 break; 00667 } 00668 case ISD::VALUETYPE: { 00669 EVT VT = cast<VTSDNode>(N)->getVT(); 00670 if (VT.isExtended()) { 00671 Erased = ExtendedValueTypeNodes.erase(VT); 00672 } else { 00673 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 00674 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 00675 } 00676 break; 00677 } 00678 default: 00679 // Remove it from the CSE Map. 00680 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 00681 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 00682 Erased = CSEMap.RemoveNode(N); 00683 break; 00684 } 00685 #ifndef NDEBUG 00686 // Verify that the node was actually in one of the CSE maps, unless it has a 00687 // flag result (which cannot be CSE'd) or is one of the special cases that are 00688 // not subject to CSE. 00689 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 00690 !N->isMachineOpcode() && !doNotCSE(N)) { 00691 N->dump(this); 00692 dbgs() << "\n"; 00693 llvm_unreachable("Node is not in map!"); 00694 } 00695 #endif 00696 return Erased; 00697 } 00698 00699 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 00700 /// maps and modified in place. Add it back to the CSE maps, unless an identical 00701 /// node already exists, in which case transfer all its users to the existing 00702 /// node. This transfer can potentially trigger recursive merging. 00703 /// 00704 void 00705 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 00706 // For node types that aren't CSE'd, just act as if no identical node 00707 // already exists. 00708 if (!doNotCSE(N)) { 00709 SDNode *Existing = CSEMap.GetOrInsertNode(N); 00710 if (Existing != N) { 00711 // If there was already an existing matching node, use ReplaceAllUsesWith 00712 // to replace the dead one with the existing one. This can cause 00713 // recursive merging of other unrelated nodes down the line. 00714 ReplaceAllUsesWith(N, Existing); 00715 00716 // N is now dead. Inform the listeners and delete it. 00717 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 00718 DUL->NodeDeleted(N, Existing); 00719 DeleteNodeNotInCSEMaps(N); 00720 return; 00721 } 00722 } 00723 00724 // If the node doesn't already exist, we updated it. Inform listeners. 00725 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 00726 DUL->NodeUpdated(N); 00727 } 00728 00729 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 00730 /// were replaced with those specified. If this node is never memoized, 00731 /// return null, otherwise return a pointer to the slot it would take. If a 00732 /// node already exists with these operands, the slot will be non-null. 00733 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 00734 void *&InsertPos) { 00735 if (doNotCSE(N)) 00736 return 0; 00737 00738 SDValue Ops[] = { Op }; 00739 FoldingSetNodeID ID; 00740 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 00741 AddNodeIDCustom(ID, N); 00742 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 00743 return Node; 00744 } 00745 00746 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 00747 /// were replaced with those specified. If this node is never memoized, 00748 /// return null, otherwise return a pointer to the slot it would take. If a 00749 /// node already exists with these operands, the slot will be non-null. 00750 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 00751 SDValue Op1, SDValue Op2, 00752 void *&InsertPos) { 00753 if (doNotCSE(N)) 00754 return 0; 00755 00756 SDValue Ops[] = { Op1, Op2 }; 00757 FoldingSetNodeID ID; 00758 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 00759 AddNodeIDCustom(ID, N); 00760 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 00761 return Node; 00762 } 00763 00764 00765 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 00766 /// were replaced with those specified. If this node is never memoized, 00767 /// return null, otherwise return a pointer to the slot it would take. If a 00768 /// node already exists with these operands, the slot will be non-null. 00769 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 00770 const SDValue *Ops,unsigned NumOps, 00771 void *&InsertPos) { 00772 if (doNotCSE(N)) 00773 return 0; 00774 00775 FoldingSetNodeID ID; 00776 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 00777 AddNodeIDCustom(ID, N); 00778 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 00779 return Node; 00780 } 00781 00782 #ifndef NDEBUG 00783 /// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 00784 static void VerifyNodeCommon(SDNode *N) { 00785 switch (N->getOpcode()) { 00786 default: 00787 break; 00788 case ISD::BUILD_PAIR: { 00789 EVT VT = N->getValueType(0); 00790 assert(N->getNumValues() == 1 && "Too many results!"); 00791 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 00792 "Wrong return type!"); 00793 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 00794 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 00795 "Mismatched operand types!"); 00796 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 00797 "Wrong operand type!"); 00798 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 00799 "Wrong return type size"); 00800 break; 00801 } 00802 case ISD::BUILD_VECTOR: { 00803 assert(N->getNumValues() == 1 && "Too many results!"); 00804 assert(N->getValueType(0).isVector() && "Wrong return type!"); 00805 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 00806 "Wrong number of operands!"); 00807 EVT EltVT = N->getValueType(0).getVectorElementType(); 00808 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 00809 assert((I->getValueType() == EltVT || 00810 (EltVT.isInteger() && I->getValueType().isInteger() && 00811 EltVT.bitsLE(I->getValueType()))) && 00812 "Wrong operand type!"); 00813 assert(I->getValueType() == N->getOperand(0).getValueType() && 00814 "Operands must all have the same type"); 00815 } 00816 break; 00817 } 00818 } 00819 } 00820 00821 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 00822 static void VerifySDNode(SDNode *N) { 00823 // The SDNode allocators cannot be used to allocate nodes with fields that are 00824 // not present in an SDNode! 00825 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 00826 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 00827 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 00828 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 00829 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 00830 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 00831 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 00832 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 00833 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 00834 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 00835 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 00836 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 00837 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 00838 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 00839 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 00840 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 00841 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 00842 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 00843 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 00844 00845 VerifyNodeCommon(N); 00846 } 00847 00848 /// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 00849 /// invalid. 00850 static void VerifyMachineNode(SDNode *N) { 00851 // The MachineNode allocators cannot be used to allocate nodes with fields 00852 // that are not present in a MachineNode! 00853 // Currently there are no such nodes. 00854 00855 VerifyNodeCommon(N); 00856 } 00857 #endif // NDEBUG 00858 00859 /// getEVTAlignment - Compute the default alignment value for the 00860 /// given type. 00861 /// 00862 unsigned SelectionDAG::getEVTAlignment(EVT VT) const { 00863 Type *Ty = VT == MVT::iPTR ? 00864 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 00865 VT.getTypeForEVT(*getContext()); 00866 00867 return TLI.getDataLayout()->getABITypeAlignment(Ty); 00868 } 00869 00870 // EntryNode could meaningfully have debug info if we can find it... 00871 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 00872 : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), 00873 TTI(0), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(), 00874 getVTList(MVT::Other)), 00875 Root(getEntryNode()), UpdateListeners(0) { 00876 AllNodes.push_back(&EntryNode); 00877 DbgInfo = new SDDbgInfo(); 00878 } 00879 00880 void SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti) { 00881 MF = &mf; 00882 TTI = tti; 00883 Context = &mf.getFunction()->getContext(); 00884 } 00885 00886 SelectionDAG::~SelectionDAG() { 00887 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 00888 allnodes_clear(); 00889 delete DbgInfo; 00890 } 00891 00892 void SelectionDAG::allnodes_clear() { 00893 assert(&*AllNodes.begin() == &EntryNode); 00894 AllNodes.remove(AllNodes.begin()); 00895 while (!AllNodes.empty()) 00896 DeallocateNode(AllNodes.begin()); 00897 } 00898 00899 void SelectionDAG::clear() { 00900 allnodes_clear(); 00901 OperandAllocator.Reset(); 00902 CSEMap.clear(); 00903 00904 ExtendedValueTypeNodes.clear(); 00905 ExternalSymbols.clear(); 00906 TargetExternalSymbols.clear(); 00907 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 00908 static_cast<CondCodeSDNode*>(0)); 00909 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 00910 static_cast<SDNode*>(0)); 00911 00912 EntryNode.UseList = 0; 00913 AllNodes.push_back(&EntryNode); 00914 Root = getEntryNode(); 00915 DbgInfo->clear(); 00916 } 00917 00918 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 00919 return VT.bitsGT(Op.getValueType()) ? 00920 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 00921 getNode(ISD::TRUNCATE, DL, VT, Op); 00922 } 00923 00924 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 00925 return VT.bitsGT(Op.getValueType()) ? 00926 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 00927 getNode(ISD::TRUNCATE, DL, VT, Op); 00928 } 00929 00930 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 00931 return VT.bitsGT(Op.getValueType()) ? 00932 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 00933 getNode(ISD::TRUNCATE, DL, VT, Op); 00934 } 00935 00936 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) { 00937 assert(!VT.isVector() && 00938 "getZeroExtendInReg should use the vector element type instead of " 00939 "the vector type!"); 00940 if (Op.getValueType() == VT) return Op; 00941 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 00942 APInt Imm = APInt::getLowBitsSet(BitWidth, 00943 VT.getSizeInBits()); 00944 return getNode(ISD::AND, DL, Op.getValueType(), Op, 00945 getConstant(Imm, Op.getValueType())); 00946 } 00947 00948 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 00949 /// 00950 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) { 00951 EVT EltVT = VT.getScalarType(); 00952 SDValue NegOne = 00953 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 00954 return getNode(ISD::XOR, DL, VT, Val, NegOne); 00955 } 00956 00957 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 00958 EVT EltVT = VT.getScalarType(); 00959 assert((EltVT.getSizeInBits() >= 64 || 00960 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 00961 "getConstant with a uint64_t value that doesn't fit in the type!"); 00962 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 00963 } 00964 00965 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 00966 return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 00967 } 00968 00969 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 00970 assert(VT.isInteger() && "Cannot create FP integer constant!"); 00971 00972 EVT EltVT = VT.getScalarType(); 00973 const ConstantInt *Elt = &Val; 00974 00975 // In some cases the vector type is legal but the element type is illegal and 00976 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 00977 // inserted value (the type does not need to match the vector element type). 00978 // Any extra bits introduced will be truncated away. 00979 if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) == 00980 TargetLowering::TypePromoteInteger) { 00981 EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT); 00982 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 00983 Elt = ConstantInt::get(*getContext(), NewVal); 00984 } 00985 00986 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 00987 "APInt size does not match type size!"); 00988 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 00989 FoldingSetNodeID ID; 00990 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 00991 ID.AddPointer(Elt); 00992 void *IP = 0; 00993 SDNode *N = NULL; 00994 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 00995 if (!VT.isVector()) 00996 return SDValue(N, 0); 00997 00998 if (!N) { 00999 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 01000 CSEMap.InsertNode(N, IP); 01001 AllNodes.push_back(N); 01002 } 01003 01004 SDValue Result(N, 0); 01005 if (VT.isVector()) { 01006 SmallVector<SDValue, 8> Ops; 01007 Ops.assign(VT.getVectorNumElements(), Result); 01008 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 01009 } 01010 return Result; 01011 } 01012 01013 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 01014 return getConstant(Val, TLI.getPointerTy(), isTarget); 01015 } 01016 01017 01018 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 01019 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 01020 } 01021 01022 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 01023 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 01024 01025 EVT EltVT = VT.getScalarType(); 01026 01027 // Do the map lookup using the actual bit pattern for the floating point 01028 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 01029 // we don't have issues with SNANs. 01030 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 01031 FoldingSetNodeID ID; 01032 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 01033 ID.AddPointer(&V); 01034 void *IP = 0; 01035 SDNode *N = NULL; 01036 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 01037 if (!VT.isVector()) 01038 return SDValue(N, 0); 01039 01040 if (!N) { 01041 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 01042 CSEMap.InsertNode(N, IP); 01043 AllNodes.push_back(N); 01044 } 01045 01046 SDValue Result(N, 0); 01047 if (VT.isVector()) { 01048 SmallVector<SDValue, 8> Ops; 01049 Ops.assign(VT.getVectorNumElements(), Result); 01050 // FIXME SDLoc info might be appropriate here 01051 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 01052 } 01053 return Result; 01054 } 01055 01056 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 01057 EVT EltVT = VT.getScalarType(); 01058 if (EltVT==MVT::f32) 01059 return getConstantFP(APFloat((float)Val), VT, isTarget); 01060 else if (EltVT==MVT::f64) 01061 return getConstantFP(APFloat(Val), VT, isTarget); 01062 else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || 01063 EltVT==MVT::f16) { 01064 bool ignored; 01065 APFloat apf = APFloat(Val); 01066 apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 01067 &ignored); 01068 return getConstantFP(apf, VT, isTarget); 01069 } else 01070 llvm_unreachable("Unsupported type in getConstantFP"); 01071 } 01072 01073 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL, 01074 EVT VT, int64_t Offset, 01075 bool isTargetGA, 01076 unsigned char TargetFlags) { 01077 assert((TargetFlags == 0 || isTargetGA) && 01078 "Cannot set target flags on target-independent globals"); 01079 01080 // Truncate (with sign-extension) the offset value to the pointer size. 01081 unsigned BitWidth = TLI.getPointerTy().getSizeInBits(); 01082 if (BitWidth < 64) 01083 Offset = SignExtend64(Offset, BitWidth); 01084 01085 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 01086 if (!GVar) { 01087 // If GV is an alias then use the aliasee for determining thread-localness. 01088 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 01089 GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 01090 } 01091 01092 unsigned Opc; 01093 if (GVar && GVar->isThreadLocal()) 01094 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 01095 else 01096 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 01097 01098 FoldingSetNodeID ID; 01099 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 01100 ID.AddPointer(GV); 01101 ID.AddInteger(Offset); 01102 ID.AddInteger(TargetFlags); 01103 ID.AddInteger(GV->getType()->getAddressSpace()); 01104 void *IP = 0; 01105 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01106 return SDValue(E, 0); 01107 01108 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(), 01109 DL.getDebugLoc(), GV, VT, 01110 Offset, TargetFlags); 01111 CSEMap.InsertNode(N, IP); 01112 AllNodes.push_back(N); 01113 return SDValue(N, 0); 01114 } 01115 01116 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 01117 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 01118 FoldingSetNodeID ID; 01119 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 01120 ID.AddInteger(FI); 01121 void *IP = 0; 01122 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01123 return SDValue(E, 0); 01124 01125 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 01126 CSEMap.InsertNode(N, IP); 01127 AllNodes.push_back(N); 01128 return SDValue(N, 0); 01129 } 01130 01131 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 01132 unsigned char TargetFlags) { 01133 assert((TargetFlags == 0 || isTarget) && 01134 "Cannot set target flags on target-independent jump tables"); 01135 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 01136 FoldingSetNodeID ID; 01137 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 01138 ID.AddInteger(JTI); 01139 ID.AddInteger(TargetFlags); 01140 void *IP = 0; 01141 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01142 return SDValue(E, 0); 01143 01144 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 01145 TargetFlags); 01146 CSEMap.InsertNode(N, IP); 01147 AllNodes.push_back(N); 01148 return SDValue(N, 0); 01149 } 01150 01151 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 01152 unsigned Alignment, int Offset, 01153 bool isTarget, 01154 unsigned char TargetFlags) { 01155 assert((TargetFlags == 0 || isTarget) && 01156 "Cannot set target flags on target-independent globals"); 01157 if (Alignment == 0) 01158 Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType()); 01159 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 01160 FoldingSetNodeID ID; 01161 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 01162 ID.AddInteger(Alignment); 01163 ID.AddInteger(Offset); 01164 ID.AddPointer(C); 01165 ID.AddInteger(TargetFlags); 01166 void *IP = 0; 01167 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01168 return SDValue(E, 0); 01169 01170 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 01171 Alignment, TargetFlags); 01172 CSEMap.InsertNode(N, IP); 01173 AllNodes.push_back(N); 01174 return SDValue(N, 0); 01175 } 01176 01177 01178 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 01179 unsigned Alignment, int Offset, 01180 bool isTarget, 01181 unsigned char TargetFlags) { 01182 assert((TargetFlags == 0 || isTarget) && 01183 "Cannot set target flags on target-independent globals"); 01184 if (Alignment == 0) 01185 Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType()); 01186 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 01187 FoldingSetNodeID ID; 01188 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 01189 ID.AddInteger(Alignment); 01190 ID.AddInteger(Offset); 01191 C->addSelectionDAGCSEId(ID); 01192 ID.AddInteger(TargetFlags); 01193 void *IP = 0; 01194 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01195 return SDValue(E, 0); 01196 01197 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 01198 Alignment, TargetFlags); 01199 CSEMap.InsertNode(N, IP); 01200 AllNodes.push_back(N); 01201 return SDValue(N, 0); 01202 } 01203 01204 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 01205 unsigned char TargetFlags) { 01206 FoldingSetNodeID ID; 01207 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0); 01208 ID.AddInteger(Index); 01209 ID.AddInteger(Offset); 01210 ID.AddInteger(TargetFlags); 01211 void *IP = 0; 01212 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01213 return SDValue(E, 0); 01214 01215 SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, 01216 TargetFlags); 01217 CSEMap.InsertNode(N, IP); 01218 AllNodes.push_back(N); 01219 return SDValue(N, 0); 01220 } 01221 01222 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 01223 FoldingSetNodeID ID; 01224 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 01225 ID.AddPointer(MBB); 01226 void *IP = 0; 01227 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01228 return SDValue(E, 0); 01229 01230 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 01231 CSEMap.InsertNode(N, IP); 01232 AllNodes.push_back(N); 01233 return SDValue(N, 0); 01234 } 01235 01236 SDValue SelectionDAG::getValueType(EVT VT) { 01237 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 01238 ValueTypeNodes.size()) 01239 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 01240 01241 SDNode *&N = VT.isExtended() ? 01242 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 01243 01244 if (N) return SDValue(N, 0); 01245 N = new (NodeAllocator) VTSDNode(VT); 01246 AllNodes.push_back(N); 01247 return SDValue(N, 0); 01248 } 01249 01250 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 01251 SDNode *&N = ExternalSymbols[Sym]; 01252 if (N) return SDValue(N, 0); 01253 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 01254 AllNodes.push_back(N); 01255 return SDValue(N, 0); 01256 } 01257 01258 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 01259 unsigned char TargetFlags) { 01260 SDNode *&N = 01261 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 01262 TargetFlags)]; 01263 if (N) return SDValue(N, 0); 01264 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 01265 AllNodes.push_back(N); 01266 return SDValue(N, 0); 01267 } 01268 01269 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 01270 if ((unsigned)Cond >= CondCodeNodes.size()) 01271 CondCodeNodes.resize(Cond+1); 01272 01273 if (CondCodeNodes[Cond] == 0) { 01274 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 01275 CondCodeNodes[Cond] = N; 01276 AllNodes.push_back(N); 01277 } 01278 01279 return SDValue(CondCodeNodes[Cond], 0); 01280 } 01281 01282 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 01283 // the shuffle mask M that point at N1 to point at N2, and indices that point 01284 // N2 to point at N1. 01285 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 01286 std::swap(N1, N2); 01287 int NElts = M.size(); 01288 for (int i = 0; i != NElts; ++i) { 01289 if (M[i] >= NElts) 01290 M[i] -= NElts; 01291 else if (M[i] >= 0) 01292 M[i] += NElts; 01293 } 01294 } 01295 01296 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, 01297 SDValue N2, const int *Mask) { 01298 assert(N1.getValueType() == N2.getValueType() && "Invalid VECTOR_SHUFFLE"); 01299 assert(VT.isVector() && N1.getValueType().isVector() && 01300 "Vector Shuffle VTs must be a vectors"); 01301 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() 01302 && "Vector Shuffle VTs must have same element type"); 01303 01304 // Canonicalize shuffle undef, undef -> undef 01305 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 01306 return getUNDEF(VT); 01307 01308 // Validate that all indices in Mask are within the range of the elements 01309 // input to the shuffle. 01310 unsigned NElts = VT.getVectorNumElements(); 01311 SmallVector<int, 8> MaskVec; 01312 for (unsigned i = 0; i != NElts; ++i) { 01313 assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 01314 MaskVec.push_back(Mask[i]); 01315 } 01316 01317 // Canonicalize shuffle v, v -> v, undef 01318 if (N1 == N2) { 01319 N2 = getUNDEF(VT); 01320 for (unsigned i = 0; i != NElts; ++i) 01321 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 01322 } 01323 01324 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 01325 if (N1.getOpcode() == ISD::UNDEF) 01326 commuteShuffle(N1, N2, MaskVec); 01327 01328 // Canonicalize all index into lhs, -> shuffle lhs, undef 01329 // Canonicalize all index into rhs, -> shuffle rhs, undef 01330 bool AllLHS = true, AllRHS = true; 01331 bool N2Undef = N2.getOpcode() == ISD::UNDEF; 01332 for (unsigned i = 0; i != NElts; ++i) { 01333 if (MaskVec[i] >= (int)NElts) { 01334 if (N2Undef) 01335 MaskVec[i] = -1; 01336 else 01337 AllLHS = false; 01338 } else if (MaskVec[i] >= 0) { 01339 AllRHS = false; 01340 } 01341 } 01342 if (AllLHS && AllRHS) 01343 return getUNDEF(VT); 01344 if (AllLHS && !N2Undef) 01345 N2 = getUNDEF(VT); 01346 if (AllRHS) { 01347 N1 = getUNDEF(VT); 01348 commuteShuffle(N1, N2, MaskVec); 01349 } 01350 01351 // If Identity shuffle, or all shuffle in to undef, return that node. 01352 bool AllUndef = true; 01353 bool Identity = true; 01354 for (unsigned i = 0; i != NElts; ++i) { 01355 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 01356 if (MaskVec[i] >= 0) AllUndef = false; 01357 } 01358 if (Identity && NElts == N1.getValueType().getVectorNumElements()) 01359 return N1; 01360 if (AllUndef) 01361 return getUNDEF(VT); 01362 01363 FoldingSetNodeID ID; 01364 SDValue Ops[2] = { N1, N2 }; 01365 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 01366 for (unsigned i = 0; i != NElts; ++i) 01367 ID.AddInteger(MaskVec[i]); 01368 01369 void* IP = 0; 01370 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01371 return SDValue(E, 0); 01372 01373 // Allocate the mask array for the node out of the BumpPtrAllocator, since 01374 // SDNode doesn't have access to it. This memory will be "leaked" when 01375 // the node is deallocated, but recovered when the NodeAllocator is released. 01376 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 01377 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 01378 01379 ShuffleVectorSDNode *N = 01380 new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(), dl.getDebugLoc(), N1, N2, MaskAlloc); 01381 CSEMap.InsertNode(N, IP); 01382 AllNodes.push_back(N); 01383 return SDValue(N, 0); 01384 } 01385 01386 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, 01387 SDValue Val, SDValue DTy, 01388 SDValue STy, SDValue Rnd, SDValue Sat, 01389 ISD::CvtCode Code) { 01390 // If the src and dest types are the same and the conversion is between 01391 // integer types of the same sign or two floats, no conversion is necessary. 01392 if (DTy == STy && 01393 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 01394 return Val; 01395 01396 FoldingSetNodeID ID; 01397 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 01398 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 01399 void* IP = 0; 01400 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01401 return SDValue(E, 0); 01402 01403 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(), dl.getDebugLoc(), Ops, 5, 01404 Code); 01405 CSEMap.InsertNode(N, IP); 01406 AllNodes.push_back(N); 01407 return SDValue(N, 0); 01408 } 01409 01410 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 01411 FoldingSetNodeID ID; 01412 AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 01413 ID.AddInteger(RegNo); 01414 void *IP = 0; 01415 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01416 return SDValue(E, 0); 01417 01418 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 01419 CSEMap.InsertNode(N, IP); 01420 AllNodes.push_back(N); 01421 return SDValue(N, 0); 01422 } 01423 01424 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 01425 FoldingSetNodeID ID; 01426 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 01427 ID.AddPointer(RegMask); 01428 void *IP = 0; 01429 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01430 return SDValue(E, 0); 01431 01432 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 01433 CSEMap.InsertNode(N, IP); 01434 AllNodes.push_back(N); 01435 return SDValue(N, 0); 01436 } 01437 01438 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { 01439 FoldingSetNodeID ID; 01440 SDValue Ops[] = { Root }; 01441 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 01442 ID.AddPointer(Label); 01443 void *IP = 0; 01444 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01445 return SDValue(E, 0); 01446 01447 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(), dl.getDebugLoc(), Root, Label); 01448 CSEMap.InsertNode(N, IP); 01449 AllNodes.push_back(N); 01450 return SDValue(N, 0); 01451 } 01452 01453 01454 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 01455 int64_t Offset, 01456 bool isTarget, 01457 unsigned char TargetFlags) { 01458 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 01459 01460 FoldingSetNodeID ID; 01461 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 01462 ID.AddPointer(BA); 01463 ID.AddInteger(Offset); 01464 ID.AddInteger(TargetFlags); 01465 void *IP = 0; 01466 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01467 return SDValue(E, 0); 01468 01469 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset, 01470 TargetFlags); 01471 CSEMap.InsertNode(N, IP); 01472 AllNodes.push_back(N); 01473 return SDValue(N, 0); 01474 } 01475 01476 SDValue SelectionDAG::getSrcValue(const Value *V) { 01477 assert((!V || V->getType()->isPointerTy()) && 01478 "SrcValue is not a pointer?"); 01479 01480 FoldingSetNodeID ID; 01481 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 01482 ID.AddPointer(V); 01483 01484 void *IP = 0; 01485 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01486 return SDValue(E, 0); 01487 01488 SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 01489 CSEMap.InsertNode(N, IP); 01490 AllNodes.push_back(N); 01491 return SDValue(N, 0); 01492 } 01493 01494 /// getMDNode - Return an MDNodeSDNode which holds an MDNode. 01495 SDValue SelectionDAG::getMDNode(const MDNode *MD) { 01496 FoldingSetNodeID ID; 01497 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 01498 ID.AddPointer(MD); 01499 01500 void *IP = 0; 01501 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 01502 return SDValue(E, 0); 01503 01504 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 01505 CSEMap.InsertNode(N, IP); 01506 AllNodes.push_back(N); 01507 return SDValue(N, 0); 01508 } 01509 01510 01511 /// getShiftAmountOperand - Return the specified value casted to 01512 /// the target's desired shift amount type. 01513 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 01514 EVT OpTy = Op.getValueType(); 01515 EVT ShTy = TLI.getShiftAmountTy(LHSTy); 01516 if (OpTy == ShTy || OpTy.isVector()) return Op; 01517 01518 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 01519 return getNode(Opcode, SDLoc(Op), ShTy, Op); 01520 } 01521 01522 /// CreateStackTemporary - Create a stack temporary, suitable for holding the 01523 /// specified value type. 01524 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 01525 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 01526 unsigned ByteSize = VT.getStoreSize(); 01527 Type *Ty = VT.getTypeForEVT(*getContext()); 01528 unsigned StackAlign = 01529 std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty), minAlign); 01530 01531 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 01532 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 01533 } 01534 01535 /// CreateStackTemporary - Create a stack temporary suitable for holding 01536 /// either of the specified value types. 01537 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 01538 unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 01539 VT2.getStoreSizeInBits())/8; 01540 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 01541 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 01542 const DataLayout *TD = TLI.getDataLayout(); 01543 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 01544 TD->getPrefTypeAlignment(Ty2)); 01545 01546 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 01547 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 01548 return getFrameIndex(FrameIdx, TLI.getPointerTy()); 01549 } 01550 01551 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 01552 SDValue N2, ISD::CondCode Cond, SDLoc dl) { 01553 // These setcc operations always fold. 01554 switch (Cond) { 01555 default: break; 01556 case ISD::SETFALSE: 01557 case ISD::SETFALSE2: return getConstant(0, VT); 01558 case ISD::SETTRUE: 01559 case ISD::SETTRUE2: return getConstant(1, VT); 01560 01561 case ISD::SETOEQ: 01562 case ISD::SETOGT: 01563 case ISD::SETOGE: 01564 case ISD::SETOLT: 01565 case ISD::SETOLE: 01566 case ISD::SETONE: 01567 case ISD::SETO: 01568 case ISD::SETUO: 01569 case ISD::SETUEQ: 01570 case ISD::SETUNE: 01571 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 01572 break; 01573 } 01574 01575 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 01576 const APInt &C2 = N2C->getAPIntValue(); 01577 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 01578 const APInt &C1 = N1C->getAPIntValue(); 01579 01580 switch (Cond) { 01581 default: llvm_unreachable("Unknown integer setcc!"); 01582 case ISD::SETEQ: return getConstant(C1 == C2, VT); 01583 case ISD::SETNE: return getConstant(C1 != C2, VT); 01584 case ISD::SETULT: return getConstant(C1.ult(C2), VT); 01585 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 01586 case ISD::SETULE: return getConstant(C1.ule(C2), VT); 01587 case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 01588 case ISD::SETLT: return getConstant(C1.slt(C2), VT); 01589 case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 01590 case ISD::SETLE: return getConstant(C1.sle(C2), VT); 01591 case ISD::SETGE: return getConstant(C1.sge(C2), VT); 01592 } 01593 } 01594 } 01595 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 01596 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 01597 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 01598 switch (Cond) { 01599 default: break; 01600 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 01601 return getUNDEF(VT); 01602 // fall through 01603 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 01604 case ISD::SETNE: if (R==APFloat::cmpUnordered) 01605 return getUNDEF(VT); 01606 // fall through 01607 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 01608 R==APFloat::cmpLessThan, VT); 01609 case ISD::SETLT: if (R==APFloat::cmpUnordered) 01610 return getUNDEF(VT); 01611 // fall through 01612 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 01613 case ISD::SETGT: if (R==APFloat::cmpUnordered) 01614 return getUNDEF(VT); 01615 // fall through 01616 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 01617 case ISD::SETLE: if (R==APFloat::cmpUnordered) 01618 return getUNDEF(VT); 01619 // fall through 01620 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 01621 R==APFloat::cmpEqual, VT); 01622 case ISD::SETGE: if (R==APFloat::cmpUnordered) 01623 return getUNDEF(VT); 01624 // fall through 01625 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 01626 R==APFloat::cmpEqual, VT); 01627 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 01628 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 01629 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 01630 R==APFloat::cmpEqual, VT); 01631 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 01632 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 01633 R==APFloat::cmpLessThan, VT); 01634 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 01635 R==APFloat::cmpUnordered, VT); 01636 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 01637 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 01638 } 01639 } else { 01640 // Ensure that the constant occurs on the RHS. 01641 return getSetCC(dl, VT, N2, N1, ISD::getSetCCSwappedOperands(Cond)); 01642 } 01643 } 01644 01645 // Could not fold it. 01646 return SDValue(); 01647 } 01648 01649 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 01650 /// use this predicate to simplify operations downstream. 01651 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 01652 // This predicate is not safe for vector operations. 01653 if (Op.getValueType().isVector()) 01654 return false; 01655 01656 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 01657 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 01658 } 01659 01660 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 01661 /// this predicate to simplify operations downstream. Mask is known to be zero 01662 /// for bits that V cannot have. 01663 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 01664 unsigned Depth) const { 01665 APInt KnownZero, KnownOne; 01666 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 01667 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01668 return (KnownZero & Mask) == Mask; 01669 } 01670 01671 /// ComputeMaskedBits - Determine which of the bits specified in Mask are 01672 /// known to be either zero or one and return them in the KnownZero/KnownOne 01673 /// bitsets. This code only analyzes bits in Mask, in order to short-circuit 01674 /// processing. 01675 void SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, 01676 APInt &KnownOne, unsigned Depth) const { 01677 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 01678 01679 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 01680 if (Depth == 6) 01681 return; // Limit search depth. 01682 01683 APInt KnownZero2, KnownOne2; 01684 01685 switch (Op.getOpcode()) { 01686 case ISD::Constant: 01687 // We know all of the bits for a constant! 01688 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue(); 01689 KnownZero = ~KnownOne; 01690 return; 01691 case ISD::AND: 01692 // If either the LHS or the RHS are Zero, the result is zero. 01693 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 01694 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 01695 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01696 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 01697 01698 // Output known-1 bits are only known if set in both the LHS & RHS. 01699 KnownOne &= KnownOne2; 01700 // Output known-0 are known to be clear if zero in either the LHS | RHS. 01701 KnownZero |= KnownZero2; 01702 return; 01703 case ISD::OR: 01704 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 01705 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 01706 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01707 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 01708 01709 // Output known-0 bits are only known if clear in both the LHS & RHS. 01710 KnownZero &= KnownZero2; 01711 // Output known-1 are known to be set if set in either the LHS | RHS. 01712 KnownOne |= KnownOne2; 01713 return; 01714 case ISD::XOR: { 01715 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 01716 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 01717 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01718 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 01719 01720 // Output known-0 bits are known if clear or set in both the LHS & RHS. 01721 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 01722 // Output known-1 are known to be set if set in only one of the LHS, RHS. 01723 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 01724 KnownZero = KnownZeroOut; 01725 return; 01726 } 01727 case ISD::MUL: { 01728 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 01729 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 01730 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01731 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 01732 01733 // If low bits are zero in either operand, output low known-0 bits. 01734 // Also compute a conserative estimate for high known-0 bits. 01735 // More trickiness is possible, but this is sufficient for the 01736 // interesting case of alignment computation. 01737 KnownOne.clearAllBits(); 01738 unsigned TrailZ = KnownZero.countTrailingOnes() + 01739 KnownZero2.countTrailingOnes(); 01740 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 01741 KnownZero2.countLeadingOnes(), 01742 BitWidth) - BitWidth; 01743 01744 TrailZ = std::min(TrailZ, BitWidth); 01745 LeadZ = std::min(LeadZ, BitWidth); 01746 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 01747 APInt::getHighBitsSet(BitWidth, LeadZ); 01748 return; 01749 } 01750 case ISD::UDIV: { 01751 // For the purposes of computing leading zeros we can conservatively 01752 // treat a udiv as a logical right shift by the power of 2 known to 01753 // be less than the denominator. 01754 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 01755 unsigned LeadZ = KnownZero2.countLeadingOnes(); 01756 01757 KnownOne2.clearAllBits(); 01758 KnownZero2.clearAllBits(); 01759 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 01760 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 01761 if (RHSUnknownLeadingOnes != BitWidth) 01762 LeadZ = std::min(BitWidth, 01763 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 01764 01765 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ); 01766 return; 01767 } 01768 case ISD::SELECT: 01769 ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1); 01770 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 01771 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01772 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 01773 01774 // Only known if known in both the LHS and RHS. 01775 KnownOne &= KnownOne2; 01776 KnownZero &= KnownZero2; 01777 return; 01778 case ISD::SELECT_CC: 01779 ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1); 01780 ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1); 01781 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01782 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 01783 01784 // Only known if known in both the LHS and RHS. 01785 KnownOne &= KnownOne2; 01786 KnownZero &= KnownZero2; 01787 return; 01788 case ISD::SADDO: 01789 case ISD::UADDO: 01790 case ISD::SSUBO: 01791 case ISD::USUBO: 01792 case ISD::SMULO: 01793 case ISD::UMULO: 01794 if (Op.getResNo() != 1) 01795 return; 01796 // The boolean result conforms to getBooleanContents. Fall through. 01797 case ISD::SETCC: 01798 // If we know the result of a setcc has the top bits zero, use this info. 01799 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 01800 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 01801 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 01802 return; 01803 case ISD::SHL: 01804 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 01805 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 01806 unsigned ShAmt = SA->getZExtValue(); 01807 01808 // If the shift count is an invalid immediate, don't do anything. 01809 if (ShAmt >= BitWidth) 01810 return; 01811 01812 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01813 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01814 KnownZero <<= ShAmt; 01815 KnownOne <<= ShAmt; 01816 // low bits known zero. 01817 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 01818 } 01819 return; 01820 case ISD::SRL: 01821 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 01822 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 01823 unsigned ShAmt = SA->getZExtValue(); 01824 01825 // If the shift count is an invalid immediate, don't do anything. 01826 if (ShAmt >= BitWidth) 01827 return; 01828 01829 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01830 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01831 KnownZero = KnownZero.lshr(ShAmt); 01832 KnownOne = KnownOne.lshr(ShAmt); 01833 01834 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 01835 KnownZero |= HighBits; // High bits known zero. 01836 } 01837 return; 01838 case ISD::SRA: 01839 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 01840 unsigned ShAmt = SA->getZExtValue(); 01841 01842 // If the shift count is an invalid immediate, don't do anything. 01843 if (ShAmt >= BitWidth) 01844 return; 01845 01846 // If any of the demanded bits are produced by the sign extension, we also 01847 // demand the input sign bit. 01848 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 01849 01850 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01851 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01852 KnownZero = KnownZero.lshr(ShAmt); 01853 KnownOne = KnownOne.lshr(ShAmt); 01854 01855 // Handle the sign bits. 01856 APInt SignBit = APInt::getSignBit(BitWidth); 01857 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 01858 01859 if (KnownZero.intersects(SignBit)) { 01860 KnownZero |= HighBits; // New bits are known zero. 01861 } else if (KnownOne.intersects(SignBit)) { 01862 KnownOne |= HighBits; // New bits are known one. 01863 } 01864 } 01865 return; 01866 case ISD::SIGN_EXTEND_INREG: { 01867 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 01868 unsigned EBits = EVT.getScalarType().getSizeInBits(); 01869 01870 // Sign extension. Compute the demanded bits in the result that are not 01871 // present in the input. 01872 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 01873 01874 APInt InSignBit = APInt::getSignBit(EBits); 01875 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 01876 01877 // If the sign extended bits are demanded, we know that the sign 01878 // bit is demanded. 01879 InSignBit = InSignBit.zext(BitWidth); 01880 if (NewBits.getBoolValue()) 01881 InputDemandedBits |= InSignBit; 01882 01883 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01884 KnownOne &= InputDemandedBits; 01885 KnownZero &= InputDemandedBits; 01886 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01887 01888 // If the sign bit of the input is known set or clear, then we know the 01889 // top bits of the result. 01890 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 01891 KnownZero |= NewBits; 01892 KnownOne &= ~NewBits; 01893 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 01894 KnownOne |= NewBits; 01895 KnownZero &= ~NewBits; 01896 } else { // Input sign bit unknown 01897 KnownZero &= ~NewBits; 01898 KnownOne &= ~NewBits; 01899 } 01900 return; 01901 } 01902 case ISD::CTTZ: 01903 case ISD::CTTZ_ZERO_UNDEF: 01904 case ISD::CTLZ: 01905 case ISD::CTLZ_ZERO_UNDEF: 01906 case ISD::CTPOP: { 01907 unsigned LowBits = Log2_32(BitWidth)+1; 01908 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 01909 KnownOne.clearAllBits(); 01910 return; 01911 } 01912 case ISD::LOAD: { 01913 LoadSDNode *LD = cast<LoadSDNode>(Op); 01914 // If this is a ZEXTLoad and we are looking at the loaded value. 01915 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 01916 EVT VT = LD->getMemoryVT(); 01917 unsigned MemBits = VT.getScalarType().getSizeInBits(); 01918 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 01919 } else if (const MDNode *Ranges = LD->getRanges()) { 01920 computeMaskedBitsLoad(*Ranges, KnownZero); 01921 } 01922 return; 01923 } 01924 case ISD::ZERO_EXTEND: { 01925 EVT InVT = Op.getOperand(0).getValueType(); 01926 unsigned InBits = InVT.getScalarType().getSizeInBits(); 01927 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 01928 KnownZero = KnownZero.trunc(InBits); 01929 KnownOne = KnownOne.trunc(InBits); 01930 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01931 KnownZero = KnownZero.zext(BitWidth); 01932 KnownOne = KnownOne.zext(BitWidth); 01933 KnownZero |= NewBits; 01934 return; 01935 } 01936 case ISD::SIGN_EXTEND: { 01937 EVT InVT = Op.getOperand(0).getValueType(); 01938 unsigned InBits = InVT.getScalarType().getSizeInBits(); 01939 APInt InSignBit = APInt::getSignBit(InBits); 01940 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 01941 01942 KnownZero = KnownZero.trunc(InBits); 01943 KnownOne = KnownOne.trunc(InBits); 01944 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01945 01946 // Note if the sign bit is known to be zero or one. 01947 bool SignBitKnownZero = KnownZero.isNegative(); 01948 bool SignBitKnownOne = KnownOne.isNegative(); 01949 assert(!(SignBitKnownZero && SignBitKnownOne) && 01950 "Sign bit can't be known to be both zero and one!"); 01951 01952 KnownZero = KnownZero.zext(BitWidth); 01953 KnownOne = KnownOne.zext(BitWidth); 01954 01955 // If the sign bit is known zero or one, the top bits match. 01956 if (SignBitKnownZero) 01957 KnownZero |= NewBits; 01958 else if (SignBitKnownOne) 01959 KnownOne |= NewBits; 01960 return; 01961 } 01962 case ISD::ANY_EXTEND: { 01963 EVT InVT = Op.getOperand(0).getValueType(); 01964 unsigned InBits = InVT.getScalarType().getSizeInBits(); 01965 KnownZero = KnownZero.trunc(InBits); 01966 KnownOne = KnownOne.trunc(InBits); 01967 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01968 KnownZero = KnownZero.zext(BitWidth); 01969 KnownOne = KnownOne.zext(BitWidth); 01970 return; 01971 } 01972 case ISD::TRUNCATE: { 01973 EVT InVT = Op.getOperand(0).getValueType(); 01974 unsigned InBits = InVT.getScalarType().getSizeInBits(); 01975 KnownZero = KnownZero.zext(InBits); 01976 KnownOne = KnownOne.zext(InBits); 01977 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01978 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 01979 KnownZero = KnownZero.trunc(BitWidth); 01980 KnownOne = KnownOne.trunc(BitWidth); 01981 break; 01982 } 01983 case ISD::AssertZext: { 01984 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 01985 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 01986 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 01987 KnownZero |= (~InMask); 01988 KnownOne &= (~KnownZero); 01989 return; 01990 } 01991 case ISD::FGETSIGN: 01992 // All bits are zero except the low bit. 01993 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 01994 return; 01995 01996 case ISD::SUB: { 01997 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 01998 // We know that the top bits of C-X are clear if X contains less bits 01999 // than C (i.e. no wrap-around can happen). For example, 20-X is 02000 // positive if we can prove that X is >= 0 and < 16. 02001 if (CLHS->getAPIntValue().isNonNegative()) { 02002 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 02003 // NLZ can't be BitWidth with no sign bit 02004 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 02005 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 02006 02007 // If all of the MaskV bits are known to be zero, then we know the 02008 // output top bits are zero, because we now know that the output is 02009 // from [0-C]. 02010 if ((KnownZero2 & MaskV) == MaskV) { 02011 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 02012 // Top bits known zero. 02013 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2); 02014 } 02015 } 02016 } 02017 } 02018 // fall through 02019 case ISD::ADD: 02020 case ISD::ADDE: { 02021 // Output known-0 bits are known if clear or set in both the low clear bits 02022 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 02023 // low 3 bits clear. 02024 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 02025 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 02026 unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 02027 02028 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 02029 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 02030 KnownZeroOut = std::min(KnownZeroOut, 02031 KnownZero2.countTrailingOnes()); 02032 02033 if (Op.getOpcode() == ISD::ADD) { 02034 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 02035 return; 02036 } 02037 02038 // With ADDE, a carry bit may be added in, so we can only use this 02039 // information if we know (at least) that the low two bits are clear. We 02040 // then return to the caller that the low bit is unknown but that other bits 02041 // are known zero. 02042 if (KnownZeroOut >= 2) // ADDE 02043 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 02044 return; 02045 } 02046 case ISD::SREM: 02047 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 02048 const APInt &RA = Rem->getAPIntValue().abs(); 02049 if (RA.isPowerOf2()) { 02050 APInt LowBits = RA - 1; 02051 APInt Mask2 = LowBits | APInt::getSignBit(BitWidth); 02052 ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1); 02053 02054 // The low bits of the first operand are unchanged by the srem. 02055 KnownZero = KnownZero2 & LowBits; 02056 KnownOne = KnownOne2 & LowBits; 02057 02058 // If the first operand is non-negative or has all low bits zero, then 02059 // the upper bits are all zero. 02060 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 02061 KnownZero |= ~LowBits; 02062 02063 // If the first operand is negative and not all low bits are zero, then 02064 // the upper bits are all one. 02065 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 02066 KnownOne |= ~LowBits; 02067 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 02068 } 02069 } 02070 return; 02071 case ISD::UREM: { 02072 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 02073 const APInt &RA = Rem->getAPIntValue(); 02074 if (RA.isPowerOf2()) { 02075 APInt LowBits = (RA - 1); 02076 KnownZero |= ~LowBits; 02077 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1); 02078 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 02079 break; 02080 } 02081 } 02082 02083 // Since the result is less than or equal to either operand, any leading 02084 // zero bits in either operand must also exist in the result. 02085 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 02086 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 02087 02088 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 02089 KnownZero2.countLeadingOnes()); 02090 KnownOne.clearAllBits(); 02091 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders); 02092 return; 02093 } 02094 case ISD::FrameIndex: 02095 case ISD::TargetFrameIndex: 02096 if (unsigned Align = InferPtrAlignment(Op)) { 02097 // The low bits are known zero if the pointer is aligned. 02098 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 02099 return; 02100 } 02101 break; 02102 02103 default: 02104 if (Op.getOpcode() < ISD::BUILTIN_OP_END) 02105 break; 02106 // Fallthrough 02107 case ISD::INTRINSIC_WO_CHAIN: 02108 case ISD::INTRINSIC_W_CHAIN: 02109 case ISD::INTRINSIC_VOID: 02110 // Allow the target to implement this method for its nodes. 02111 TLI.computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth); 02112 return; 02113 } 02114 } 02115 02116 /// ComputeNumSignBits - Return the number of times the sign bit of the 02117 /// register is replicated into the other bits. We know that at least 1 bit 02118 /// is always equal to the sign bit (itself), but other cases can give us 02119 /// information. For example, immediately after an "SRA X, 2", we know that 02120 /// the top 3 bits are all equal to each other, so we return 3. 02121 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 02122 EVT VT = Op.getValueType(); 02123 assert(VT.isInteger() && "Invalid VT!"); 02124 unsigned VTBits = VT.getScalarType().getSizeInBits(); 02125 unsigned Tmp, Tmp2; 02126 unsigned FirstAnswer = 1; 02127 02128 if (Depth == 6) 02129 return 1; // Limit search depth. 02130 02131 switch (Op.getOpcode()) { 02132 default: break; 02133 case ISD::AssertSext: 02134 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 02135 return VTBits-Tmp+1; 02136 case ISD::AssertZext: 02137 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 02138 return VTBits-Tmp; 02139 02140 case ISD::Constant: { 02141 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 02142 return Val.getNumSignBits(); 02143 } 02144 02145 case ISD::SIGN_EXTEND: 02146 Tmp = VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 02147 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 02148 02149 case ISD::SIGN_EXTEND_INREG: 02150 // Max of the input and what this extends. 02151 Tmp = 02152 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 02153 Tmp = VTBits-Tmp+1; 02154 02155 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 02156 return std::max(Tmp, Tmp2); 02157 02158 case ISD::SRA: 02159 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 02160 // SRA X, C -> adds C sign bits. 02161 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 02162 Tmp += C->getZExtValue(); 02163 if (Tmp > VTBits) Tmp = VTBits; 02164 } 02165 return Tmp; 02166 case ISD::SHL: 02167 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 02168 // shl destroys sign bits. 02169 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 02170 if (C->getZExtValue() >= VTBits || // Bad shift. 02171 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 02172 return Tmp - C->getZExtValue(); 02173 } 02174 break; 02175 case ISD::AND: 02176 case ISD::OR: 02177 case ISD::XOR: // NOT is handled here. 02178 // Logical binary ops preserve the number of sign bits at the worst. 02179 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 02180 if (Tmp != 1) { 02181 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 02182 FirstAnswer = std::min(Tmp, Tmp2); 02183 // We computed what we know about the sign bits as our first 02184 // answer. Now proceed to the generic code that uses 02185 // ComputeMaskedBits, and pick whichever answer is better. 02186 } 02187 break; 02188 02189 case ISD::SELECT: 02190 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 02191 if (Tmp == 1) return 1; // Early out. 02192 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 02193 return std::min(Tmp, Tmp2); 02194 02195 case ISD::SADDO: 02196 case ISD::UADDO: 02197 case ISD::SSUBO: 02198 case ISD::USUBO: 02199 case ISD::SMULO: 02200 case ISD::UMULO: 02201 if (Op.getResNo() != 1) 02202 break; 02203 // The boolean result conforms to getBooleanContents. Fall through. 02204 case ISD::SETCC: 02205 // If setcc returns 0/-1, all bits are sign bits. 02206 if (TLI.getBooleanContents(Op.getValueType().isVector()) == 02207 TargetLowering::ZeroOrNegativeOneBooleanContent) 02208 return VTBits; 02209 break; 02210 case ISD::ROTL: 02211 case ISD::ROTR: 02212 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 02213 unsigned RotAmt = C->getZExtValue() & (VTBits-1); 02214 02215 // Handle rotate right by N like a rotate left by 32-N. 02216 if (Op.getOpcode() == ISD::ROTR) 02217 RotAmt = (VTBits-RotAmt) & (VTBits-1); 02218 02219 // If we aren't rotating out all of the known-in sign bits, return the 02220 // number that are left. This handles rotl(sext(x), 1) for example. 02221 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 02222 if (Tmp > RotAmt+1) return Tmp-RotAmt; 02223 } 02224 break; 02225 case ISD::ADD: 02226 // Add can have at most one carry bit. Thus we know that the output 02227 // is, at worst, one more bit than the inputs. 02228 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 02229 if (Tmp == 1) return 1; // Early out. 02230 02231 // Special case decrementing a value (ADD X, -1): 02232 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 02233 if (CRHS->isAllOnesValue()) { 02234 APInt KnownZero, KnownOne; 02235 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 02236 02237 // If the input is known to be 0 or 1, the output is 0/-1, which is all 02238 // sign bits set. 02239 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 02240 return VTBits; 02241 02242 // If we are subtracting one from a positive number, there is no carry 02243 // out of the result. 02244 if (KnownZero.isNegative()) 02245 return Tmp; 02246 } 02247 02248 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 02249 if (Tmp2 == 1) return 1; 02250 return std::min(Tmp, Tmp2)-1; 02251 02252 case ISD::SUB: 02253 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 02254 if (Tmp2 == 1) return 1; 02255 02256 // Handle NEG. 02257 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 02258 if (CLHS->isNullValue()) { 02259 APInt KnownZero, KnownOne; 02260 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 02261 // If the input is known to be 0 or 1, the output is 0/-1, which is all 02262 // sign bits set. 02263 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 02264 return VTBits; 02265 02266 // If the input is known to be positive (the sign bit is known clear), 02267 // the output of the NEG has the same number of sign bits as the input. 02268 if (KnownZero.isNegative()) 02269 return Tmp2; 02270 02271 // Otherwise, we treat this like a SUB. 02272 } 02273 02274 // Sub can have at most one carry bit. Thus we know that the output 02275 // is, at worst, one more bit than the inputs. 02276 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 02277 if (Tmp == 1) return 1; // Early out. 02278 return std::min(Tmp, Tmp2)-1; 02279 case ISD::TRUNCATE: 02280 // FIXME: it's tricky to do anything useful for this, but it is an important 02281 // case for targets like X86. 02282 break; 02283 } 02284 02285 // If we are looking at the loaded value of the SDNode. 02286 if (Op.getResNo() == 0) { 02287 // Handle LOADX separately here. EXTLOAD case will fallthrough. 02288 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 02289 unsigned ExtType = LD->getExtensionType(); 02290 switch (ExtType) { 02291 default: break; 02292 case ISD::SEXTLOAD: // '17' bits known 02293 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 02294 return VTBits-Tmp+1; 02295 case ISD::ZEXTLOAD: // '16' bits known 02296 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 02297 return VTBits-Tmp; 02298 } 02299 } 02300 } 02301 02302 // Allow the target to implement this method for its nodes. 02303 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 02304 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 02305 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 02306 Op.getOpcode() == ISD::INTRINSIC_VOID) { 02307 unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth); 02308 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 02309 } 02310 02311 // Finally, if we can prove that the top bits of the result are 0's or 1's, 02312 // use this information. 02313 APInt KnownZero, KnownOne; 02314 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 02315 02316 APInt Mask; 02317 if (KnownZero.isNegative()) { // sign bit is 0 02318 Mask = KnownZero; 02319 } else if (KnownOne.isNegative()) { // sign bit is 1; 02320 Mask = KnownOne; 02321 } else { 02322 // Nothing known. 02323 return FirstAnswer; 02324 } 02325 02326 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 02327 // the number of identical bits in the top of the input value. 02328 Mask = ~Mask; 02329 Mask <<= Mask.getBitWidth()-VTBits; 02330 // Return # leading zeros. We use 'min' here in case Val was zero before 02331 // shifting. We don't want to return '64' as for an i32 "0". 02332 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 02333 } 02334 02335 /// isBaseWithConstantOffset - Return true if the specified operand is an 02336 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 02337 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same 02338 /// semantics as an ADD. This handles the equivalence: 02339 /// X|Cst == X+Cst iff X&Cst = 0. 02340 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 02341 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 02342 !isa<ConstantSDNode>(Op.getOperand(1))) 02343 return false; 02344 02345 if (Op.getOpcode() == ISD::OR && 02346 !MaskedValueIsZero(Op.getOperand(0), 02347 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 02348 return false; 02349 02350 return true; 02351 } 02352 02353 02354 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 02355 // If we're told that NaNs won't happen, assume they won't. 02356 if (getTarget().Options.NoNaNsFPMath) 02357 return true; 02358 02359 // If the value is a constant, we can obviously see if it is a NaN or not. 02360 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 02361 return !C->getValueAPF().isNaN(); 02362 02363 // TODO: Recognize more cases here. 02364 02365 return false; 02366 } 02367 02368 bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 02369 // If the value is a constant, we can obviously see if it is a zero or not. 02370 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 02371 return !C->isZero(); 02372 02373 // TODO: Recognize more cases here. 02374 switch (Op.getOpcode()) { 02375 default: break; 02376 case ISD::OR: 02377 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 02378 return !C->isNullValue(); 02379 break; 02380 } 02381 02382 return false; 02383 } 02384 02385 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 02386 // Check the obvious case. 02387 if (A == B) return true; 02388 02389 // For for negative and positive zero. 02390 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 02391 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 02392 if (CA->isZero() && CB->isZero()) return true; 02393 02394 // Otherwise they may not be equal. 02395 return false; 02396 } 02397 02398 /// getNode - Gets or creates the specified node. 02399 /// 02400 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) { 02401 FoldingSetNodeID ID; 02402 AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 02403 void *IP = 0; 02404 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 02405 return SDValue(E, 0); 02406 02407 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), getVTList(VT)); 02408 CSEMap.InsertNode(N, IP); 02409 02410 AllNodes.push_back(N); 02411 #ifndef NDEBUG 02412 VerifySDNode(N); 02413 #endif 02414 return SDValue(N, 0); 02415 } 02416 02417 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 02418 EVT VT, SDValue Operand) { 02419 // Constant fold unary operations with an integer constant operand. 02420 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 02421 const APInt &Val = C->getAPIntValue(); 02422 switch (Opcode) { 02423 default: break; 02424 case ISD::SIGN_EXTEND: 02425 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 02426 case ISD::ANY_EXTEND: 02427 case ISD::ZERO_EXTEND: 02428 case ISD::TRUNCATE: 02429 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 02430 case ISD::UINT_TO_FP: 02431 case ISD::SINT_TO_FP: { 02432 APFloat apf(EVTToAPFloatSemantics(VT), 02433 APInt::getNullValue(VT.getSizeInBits())); 02434 (void)apf.convertFromAPInt(Val, 02435 Opcode==ISD::SINT_TO_FP, 02436 APFloat::rmNearestTiesToEven); 02437 return getConstantFP(apf, VT); 02438 } 02439 case ISD::BITCAST: 02440 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 02441 return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); 02442 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 02443 return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); 02444 break; 02445 case ISD::BSWAP: 02446 return getConstant(Val.byteSwap(), VT); 02447 case ISD::CTPOP: 02448 return getConstant(Val.countPopulation(), VT); 02449 case ISD::CTLZ: 02450 case ISD::CTLZ_ZERO_UNDEF: 02451 return getConstant(Val.countLeadingZeros(), VT); 02452 case ISD::CTTZ: 02453 case ISD::CTTZ_ZERO_UNDEF: 02454 return getConstant(Val.countTrailingZeros(), VT); 02455 } 02456 } 02457 02458 // Constant fold unary operations with a floating point constant operand. 02459 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 02460 APFloat V = C->getValueAPF(); // make copy 02461 switch (Opcode) { 02462 case ISD::FNEG: 02463 V.changeSign(); 02464 return getConstantFP(V, VT); 02465 case ISD::FABS: 02466 V.clearSign(); 02467 return getConstantFP(V, VT); 02468 case ISD::FCEIL: { 02469 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 02470 if (fs == APFloat::opOK || fs == APFloat::opInexact) 02471 return getConstantFP(V, VT); 02472 break; 02473 } 02474 case ISD::FTRUNC: { 02475 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 02476 if (fs == APFloat::opOK || fs == APFloat::opInexact) 02477 return getConstantFP(V, VT); 02478 break; 02479 } 02480 case ISD::FFLOOR: { 02481 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 02482 if (fs == APFloat::opOK || fs == APFloat::opInexact) 02483 return getConstantFP(V, VT); 02484 break; 02485 } 02486 case ISD::FP_EXTEND: { 02487 bool ignored; 02488 // This can return overflow, underflow, or inexact; we don't care. 02489 // FIXME need to be more flexible about rounding mode. 02490 (void)V.convert(EVTToAPFloatSemantics(VT), 02491 APFloat::rmNearestTiesToEven, &ignored); 02492 return getConstantFP(V, VT); 02493 } 02494 case ISD::FP_TO_SINT: 02495 case ISD::FP_TO_UINT: { 02496 integerPart x[2]; 02497 bool ignored; 02498 assert(integerPartWidth >= 64); 02499 // FIXME need to be more flexible about rounding mode. 02500 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 02501 Opcode==ISD::FP_TO_SINT, 02502 APFloat::rmTowardZero, &ignored); 02503 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 02504 break; 02505 APInt api(VT.getSizeInBits(), x); 02506 return getConstant(api, VT); 02507 } 02508 case ISD::BITCAST: 02509 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 02510 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 02511 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 02512 return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 02513 break; 02514 } 02515 } 02516 02517 unsigned OpOpcode = Operand.getNode()->getOpcode(); 02518 switch (Opcode) { 02519 case ISD::TokenFactor: 02520 case ISD::MERGE_VALUES: 02521 case ISD::CONCAT_VECTORS: 02522 return Operand; // Factor, merge or concat of one node? No need. 02523 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 02524 case ISD::FP_EXTEND: 02525 assert(VT.isFloatingPoint() && 02526 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 02527 if (Operand.getValueType() == VT) return Operand; // noop conversion. 02528 assert((!VT.isVector() || 02529 VT.getVectorNumElements() == 02530 Operand.getValueType().getVectorNumElements()) && 02531 "Vector element count mismatch!"); 02532 if (Operand.getOpcode() == ISD::UNDEF) 02533 return getUNDEF(VT); 02534 break; 02535 case ISD::SIGN_EXTEND: 02536 assert(VT.isInteger() && Operand.getValueType().isInteger() && 02537 "Invalid SIGN_EXTEND!"); 02538 if (Operand.getValueType() == VT) return Operand; // noop extension 02539 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 02540 "Invalid sext node, dst < src!"); 02541 assert((!VT.isVector() || 02542 VT.getVectorNumElements() == 02543 Operand.getValueType().getVectorNumElements()) && 02544 "Vector element count mismatch!"); 02545 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 02546 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 02547 else if (OpOpcode == ISD::UNDEF) 02548 // sext(undef) = 0, because the top bits will all be the same. 02549 return getConstant(0, VT); 02550 break; 02551 case ISD::ZERO_EXTEND: 02552 assert(VT.isInteger() && Operand.getValueType().isInteger() && 02553 "Invalid ZERO_EXTEND!"); 02554 if (Operand.getValueType() == VT) return Operand; // noop extension 02555 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 02556 "Invalid zext node, dst < src!"); 02557 assert((!VT.isVector() || 02558 VT.getVectorNumElements() == 02559 Operand.getValueType().getVectorNumElements()) && 02560 "Vector element count mismatch!"); 02561 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 02562 return getNode(ISD::ZERO_EXTEND, DL, VT, 02563 Operand.getNode()->getOperand(0)); 02564 else if (OpOpcode == ISD::UNDEF) 02565 // zext(undef) = 0, because the top bits will be zero. 02566 return getConstant(0, VT); 02567 break; 02568 case ISD::ANY_EXTEND: 02569 assert(VT.isInteger() && Operand.getValueType().isInteger() && 02570 "Invalid ANY_EXTEND!"); 02571 if (Operand.getValueType() == VT) return Operand; // noop extension 02572 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 02573 "Invalid anyext node, dst < src!"); 02574 assert((!VT.isVector() || 02575 VT.getVectorNumElements() == 02576 Operand.getValueType().getVectorNumElements()) && 02577 "Vector element count mismatch!"); 02578 02579 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 02580 OpOpcode == ISD::ANY_EXTEND) 02581 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 02582 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 02583 else if (OpOpcode == ISD::UNDEF) 02584 return getUNDEF(VT); 02585 02586 // (ext (trunx x)) -> x 02587 if (OpOpcode == ISD::TRUNCATE) { 02588 SDValue OpOp = Operand.getNode()->getOperand(0); 02589 if (OpOp.getValueType() == VT) 02590 return OpOp; 02591 } 02592 break; 02593 case ISD::TRUNCATE: 02594 assert(VT.isInteger() && Operand.getValueType().isInteger() && 02595 "Invalid TRUNCATE!"); 02596 if (Operand.getValueType() == VT) return Operand; // noop truncate 02597 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 02598 "Invalid truncate node, src < dst!"); 02599 assert((!VT.isVector() || 02600 VT.getVectorNumElements() == 02601 Operand.getValueType().getVectorNumElements()) && 02602 "Vector element count mismatch!"); 02603 if (OpOpcode == ISD::TRUNCATE) 02604 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 02605 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 02606 OpOpcode == ISD::ANY_EXTEND) { 02607 // If the source is smaller than the dest, we still need an extend. 02608 if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 02609 .bitsLT(VT.getScalarType())) 02610 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 02611 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 02612 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 02613 return Operand.getNode()->getOperand(0); 02614 } 02615 if (OpOpcode == ISD::UNDEF) 02616 return getUNDEF(VT); 02617 break; 02618 case ISD::BITCAST: 02619 // Basic sanity checking. 02620 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 02621 && "Cannot BITCAST between types of different sizes!"); 02622 if (VT == Operand.getValueType()) return Operand; // noop conversion. 02623 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 02624 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 02625 if (OpOpcode == ISD::UNDEF) 02626 return getUNDEF(VT); 02627 break; 02628 case ISD::SCALAR_TO_VECTOR: 02629 assert(VT.isVector() && !Operand.getValueType().isVector() && 02630 (VT.getVectorElementType() == Operand.getValueType() || 02631 (VT.getVectorElementType().isInteger() && 02632 Operand.getValueType().isInteger() && 02633 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 02634 "Illegal SCALAR_TO_VECTOR node!"); 02635 if (OpOpcode == ISD::UNDEF) 02636 return getUNDEF(VT); 02637 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 02638 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 02639 isa<ConstantSDNode>(Operand.getOperand(1)) && 02640 Operand.getConstantOperandVal(1) == 0 && 02641 Operand.getOperand(0).getValueType() == VT) 02642 return Operand.getOperand(0); 02643 break; 02644 case ISD::FNEG: 02645 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 02646 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 02647 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 02648 Operand.getNode()->getOperand(0)); 02649 if (OpOpcode == ISD::FNEG) // --X -> X 02650 return Operand.getNode()->getOperand(0); 02651 break; 02652 case ISD::FABS: 02653 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 02654 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 02655 break; 02656 } 02657 02658 SDNode *N; 02659 SDVTList VTs = getVTList(VT); 02660 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 02661 FoldingSetNodeID ID; 02662 SDValue Ops[1] = { Operand }; 02663 AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 02664 void *IP = 0; 02665 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 02666 return SDValue(E, 0); 02667 02668 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Operand); 02669 CSEMap.InsertNode(N, IP); 02670 } else { 02671 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Operand); 02672 } 02673 02674 AllNodes.push_back(N); 02675 #ifndef NDEBUG 02676 VerifySDNode(N); 02677 #endif 02678 return SDValue(N, 0); 02679 } 02680 02681 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, 02682 SDNode *Cst1, SDNode *Cst2) { 02683 SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; 02684 SmallVector<SDValue, 4> Outputs; 02685 EVT SVT = VT.getScalarType(); 02686 02687 ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); 02688 ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); 02689 if (Scalar1 && Scalar2) { 02690 // Scalar instruction. 02691 Inputs.push_back(std::make_pair(Scalar1, Scalar2)); 02692 } else { 02693 // For vectors extract each constant element into Inputs so we can constant 02694 // fold them individually. 02695 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 02696 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 02697 if (!BV1 || !BV2) 02698 return SDValue(); 02699 02700 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 02701 02702 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 02703 ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); 02704 ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); 02705 if (!V1 || !V2) // Not a constant, bail. 02706 return SDValue(); 02707 02708 // Avoid BUILD_VECTOR nodes that perform implicit truncation. 02709 // FIXME: This is valid and could be handled by truncating the APInts. 02710 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 02711 return SDValue(); 02712 02713 Inputs.push_back(std::make_pair(V1, V2)); 02714 } 02715 } 02716 02717 // We have a number of constant values, constant fold them element by element. 02718 for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { 02719 const APInt &C1 = Inputs[I].first->getAPIntValue(); 02720 const APInt &C2 = Inputs[I].second->getAPIntValue(); 02721 02722 switch (Opcode) { 02723 case ISD::ADD: 02724 Outputs.push_back(getConstant(C1 + C2, SVT)); 02725 break; 02726 case ISD::SUB: 02727 Outputs.push_back(getConstant(C1 - C2, SVT)); 02728 break; 02729 case ISD::MUL: 02730 Outputs.push_back(getConstant(C1 * C2, SVT)); 02731 break; 02732 case ISD::UDIV: 02733 if (!C2.getBoolValue()) 02734 return SDValue(); 02735 Outputs.push_back(getConstant(C1.udiv(C2), SVT)); 02736 break; 02737 case ISD::UREM: 02738 if (!C2.getBoolValue()) 02739 return SDValue(); 02740 Outputs.push_back(getConstant(C1.urem(C2), SVT)); 02741 break; 02742 case ISD::SDIV: 02743 if (!C2.getBoolValue()) 02744 return SDValue(); 02745 Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); 02746 break; 02747 case ISD::SREM: 02748 if (!C2.getBoolValue()) 02749 return SDValue(); 02750 Outputs.push_back(getConstant(C1.srem(C2), SVT)); 02751 break; 02752 case ISD::AND: 02753 Outputs.push_back(getConstant(C1 & C2, SVT)); 02754 break; 02755 case ISD::OR: 02756 Outputs.push_back(getConstant(C1 | C2, SVT)); 02757 break; 02758 case ISD::XOR: 02759 Outputs.push_back(getConstant(C1 ^ C2, SVT)); 02760 break; 02761 case ISD::SHL: 02762 Outputs.push_back(getConstant(C1 << C2, SVT)); 02763 break; 02764 case ISD::SRL: 02765 Outputs.push_back(getConstant(C1.lshr(C2), SVT)); 02766 break; 02767 case ISD::SRA: 02768 Outputs.push_back(getConstant(C1.ashr(C2), SVT)); 02769 break; 02770 case ISD::ROTL: 02771 Outputs.push_back(getConstant(C1.rotl(C2), SVT)); 02772 break; 02773 case ISD::ROTR: 02774 Outputs.push_back(getConstant(C1.rotr(C2), SVT)); 02775 break; 02776 default: 02777 return SDValue(); 02778 } 02779 } 02780 02781 // Handle the scalar case first. 02782 if (Scalar1 && Scalar2) 02783 return Outputs.back(); 02784 02785 // Otherwise build a big vector out of the scalar elements we generated. 02786 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(), 02787 Outputs.size()); 02788 } 02789 02790 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, 02791 SDValue N2) { 02792 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 02793 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 02794 switch (Opcode) { 02795 default: break; 02796 case ISD::TokenFactor: 02797 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 02798 N2.getValueType() == MVT::Other && "Invalid token factor!"); 02799 // Fold trivial token factors. 02800 if (N1.getOpcode() == ISD::EntryToken) return N2; 02801 if (N2.getOpcode() == ISD::EntryToken) return N1; 02802 if (N1 == N2) return N1; 02803 break; 02804 case ISD::CONCAT_VECTORS: 02805 // Concat of UNDEFs is UNDEF. 02806 if (N1.getOpcode() == ISD::UNDEF && 02807 N2.getOpcode() == ISD::UNDEF) 02808 return getUNDEF(VT); 02809 02810 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 02811 // one big BUILD_VECTOR. 02812 if (N1.getOpcode() == ISD::BUILD_VECTOR && 02813 N2.getOpcode() == ISD::BUILD_VECTOR) { 02814 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 02815 N1.getNode()->op_end()); 02816 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 02817 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 02818 } 02819 break; 02820 case ISD::AND: 02821 assert(VT.isInteger() && "This operator does not apply to FP types!"); 02822 assert(N1.getValueType() == N2.getValueType() && 02823 N1.getValueType() == VT && "Binary operator types must match!"); 02824 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 02825 // worth handling here. 02826 if (N2C && N2C->isNullValue()) 02827 return N2; 02828 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 02829 return N1; 02830 break; 02831 case ISD::OR: 02832 case ISD::XOR: 02833 case ISD::ADD: 02834 case ISD::SUB: 02835 assert(VT.isInteger() && "This operator does not apply to FP types!"); 02836 assert(N1.getValueType() == N2.getValueType() && 02837 N1.getValueType() == VT && "Binary operator types must match!"); 02838 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 02839 // it's worth handling here. 02840 if (N2C && N2C->isNullValue()) 02841 return N1; 02842 break; 02843 case ISD::UDIV: 02844 case ISD::UREM: 02845 case ISD::MULHU: 02846 case ISD::MULHS: 02847 case ISD::MUL: 02848 case ISD::SDIV: 02849 case ISD::SREM: 02850 assert(VT.isInteger() && "This operator does not apply to FP types!"); 02851 assert(N1.getValueType() == N2.getValueType() && 02852 N1.getValueType() == VT && "Binary operator types must match!"); 02853 break; 02854 case ISD::FADD: 02855 case ISD::FSUB: 02856 case ISD::FMUL: 02857 case ISD::FDIV: 02858 case ISD::FREM: 02859 if (getTarget().Options.UnsafeFPMath) { 02860 if (Opcode == ISD::FADD) { 02861 // 0+x --> x 02862 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 02863 if (CFP->getValueAPF().isZero()) 02864 return N2; 02865 // x+0 --> x 02866 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 02867 if (CFP->getValueAPF().isZero()) 02868 return N1; 02869 } else if (Opcode == ISD::FSUB) { 02870 // x-0 --> x 02871 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 02872 if (CFP->getValueAPF().isZero()) 02873 return N1; 02874 } else if (Opcode == ISD::FMUL) { 02875 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1); 02876 SDValue V = N2; 02877 02878 // If the first operand isn't the constant, try the second 02879 if (!CFP) { 02880 CFP = dyn_cast<ConstantFPSDNode>(N2); 02881 V = N1; 02882 } 02883 02884 if (CFP) { 02885 // 0*x --> 0 02886 if (CFP->isZero()) 02887 return SDValue(CFP,0); 02888 // 1*x --> x 02889 if (CFP->isExactlyValue(1.0)) 02890 return V; 02891 } 02892 } 02893 } 02894 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 02895 assert(N1.getValueType() == N2.getValueType() && 02896 N1.getValueType() == VT && "Binary operator types must match!"); 02897 break; 02898 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 02899 assert(N1.getValueType() == VT && 02900 N1.getValueType().isFloatingPoint() && 02901 N2.getValueType().isFloatingPoint() && 02902 "Invalid FCOPYSIGN!"); 02903 break; 02904 case ISD::SHL: 02905 case ISD::SRA: 02906 case ISD::SRL: 02907 case ISD::ROTL: 02908 case ISD::ROTR: 02909 assert(VT == N1.getValueType() && 02910 "Shift operators return type must be the same as their first arg"); 02911 assert(VT.isInteger() && N2.getValueType().isInteger() && 02912 "Shifts only work on integers"); 02913 assert((!VT.isVector() || VT == N2.getValueType()) && 02914 "Vector shift amounts must be in the same as their first arg"); 02915 // Verify that the shift amount VT is bit enough to hold valid shift 02916 // amounts. This catches things like trying to shift an i1024 value by an 02917 // i8, which is easy to fall into in generic code that uses 02918 // TLI.getShiftAmount(). 02919 assert(N2.getValueType().getSizeInBits() >= 02920 Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 02921 "Invalid use of small shift amount with oversized value!"); 02922 02923 // Always fold shifts of i1 values so the code generator doesn't need to 02924 // handle them. Since we know the size of the shift has to be less than the 02925 // size of the value, the shift/rotate count is guaranteed to be zero. 02926 if (VT == MVT::i1) 02927 return N1; 02928 if (N2C && N2C->isNullValue()) 02929 return N1; 02930 break; 02931 case ISD::FP_ROUND_INREG: { 02932 EVT EVT = cast<VTSDNode>(N2)->getVT(); 02933 assert(VT == N1.getValueType() && "Not an inreg round!"); 02934 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 02935 "Cannot FP_ROUND_INREG integer types"); 02936 assert(EVT.isVector() == VT.isVector() && 02937 "FP_ROUND_INREG type should be vector iff the operand " 02938 "type is vector!"); 02939 assert((!EVT.isVector() || 02940 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 02941 "Vector element counts must match in FP_ROUND_INREG"); 02942 assert(EVT.bitsLE(VT) && "Not rounding down!"); 02943 (void)EVT; 02944 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 02945 break; 02946 } 02947 case ISD::FP_ROUND: 02948 assert(VT.isFloatingPoint() && 02949 N1.getValueType().isFloatingPoint() && 02950 VT.bitsLE(N1.getValueType()) && 02951 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 02952 if (N1.getValueType() == VT) return N1; // noop conversion. 02953 break; 02954 case ISD::AssertSext: 02955 case ISD::AssertZext: { 02956 EVT EVT = cast<VTSDNode>(N2)->getVT(); 02957 assert(VT == N1.getValueType() && "Not an inreg extend!"); 02958 assert(VT.isInteger() && EVT.isInteger() && 02959 "Cannot *_EXTEND_INREG FP types"); 02960 assert(!EVT.isVector() && 02961 "AssertSExt/AssertZExt type should be the vector element type " 02962 "rather than the vector type!"); 02963 assert(EVT.bitsLE(VT) && "Not extending!"); 02964 if (VT == EVT) return N1; // noop assertion. 02965 break; 02966 } 02967 case ISD::SIGN_EXTEND_INREG: { 02968 EVT EVT = cast<VTSDNode>(N2)->getVT(); 02969 assert(VT == N1.getValueType() && "Not an inreg extend!"); 02970 assert(VT.isInteger() && EVT.isInteger() && 02971 "Cannot *_EXTEND_INREG FP types"); 02972 assert(EVT.isVector() == VT.isVector() && 02973 "SIGN_EXTEND_INREG type should be vector iff the operand " 02974 "type is vector!"); 02975 assert((!EVT.isVector() || 02976 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 02977 "Vector element counts must match in SIGN_EXTEND_INREG"); 02978 assert(EVT.bitsLE(VT) && "Not extending!"); 02979 if (EVT == VT) return N1; // Not actually extending 02980 02981 if (N1C) { 02982 APInt Val = N1C->getAPIntValue(); 02983 unsigned FromBits = EVT.getScalarType().getSizeInBits(); 02984 Val <<= Val.getBitWidth()-FromBits; 02985 Val = Val.ashr(Val.getBitWidth()-FromBits); 02986 return getConstant(Val, VT); 02987 } 02988 break; 02989 } 02990 case ISD::EXTRACT_VECTOR_ELT: 02991 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 02992 if (N1.getOpcode() == ISD::UNDEF) 02993 return getUNDEF(VT); 02994 02995 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 02996 // expanding copies of large vectors from registers. 02997 if (N2C && 02998 N1.getOpcode() == ISD::CONCAT_VECTORS && 02999 N1.getNumOperands() > 0) { 03000 unsigned Factor = 03001 N1.getOperand(0).getValueType().getVectorNumElements(); 03002 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 03003 N1.getOperand(N2C->getZExtValue() / Factor), 03004 getConstant(N2C->getZExtValue() % Factor, 03005 N2.getValueType())); 03006 } 03007 03008 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 03009 // expanding large vector constants. 03010 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 03011 SDValue Elt = N1.getOperand(N2C->getZExtValue()); 03012 03013 if (VT != Elt.getValueType()) 03014 // If the vector element type is not legal, the BUILD_VECTOR operands 03015 // are promoted and implicitly truncated, and the result implicitly 03016 // extended. Make that explicit here. 03017 Elt = getAnyExtOrTrunc(Elt, DL, VT); 03018 03019 return Elt; 03020 } 03021 03022 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 03023 // operations are lowered to scalars. 03024 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 03025 // If the indices are the same, return the inserted element else 03026 // if the indices are known different, extract the element from 03027 // the original vector. 03028 SDValue N1Op2 = N1.getOperand(2); 03029 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 03030 03031 if (N1Op2C && N2C) { 03032 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 03033 if (VT == N1.getOperand(1).getValueType()) 03034 return N1.getOperand(1); 03035 else 03036 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 03037 } 03038 03039 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 03040 } 03041 } 03042 break; 03043 case ISD::EXTRACT_ELEMENT: 03044 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 03045 assert(!N1.getValueType().isVector() && !VT.isVector() && 03046 (N1.getValueType().isInteger() == VT.isInteger()) && 03047 N1.getValueType() != VT && 03048 "Wrong types for EXTRACT_ELEMENT!"); 03049 03050 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 03051 // 64-bit integers into 32-bit parts. Instead of building the extract of 03052 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 03053 if (N1.getOpcode() == ISD::BUILD_PAIR) 03054 return N1.getOperand(N2C->getZExtValue()); 03055 03056 // EXTRACT_ELEMENT of a constant int is also very common. 03057 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 03058 unsigned ElementSize = VT.getSizeInBits(); 03059 unsigned Shift = ElementSize * N2C->getZExtValue(); 03060 APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 03061 return getConstant(ShiftedVal.trunc(ElementSize), VT); 03062 } 03063 break; 03064 case ISD::EXTRACT_SUBVECTOR: { 03065 SDValue Index = N2; 03066 if (VT.isSimple() && N1.getValueType().isSimple()) { 03067 assert(VT.isVector() && N1.getValueType().isVector() && 03068 "Extract subvector VTs must be a vectors!"); 03069 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() && 03070 "Extract subvector VTs must have the same element type!"); 03071 assert(VT.getSimpleVT() <= N1.getValueType().getSimpleVT() && 03072 "Extract subvector must be from larger vector to smaller vector!"); 03073 03074 if (isa<ConstantSDNode>(Index.getNode())) { 03075 assert((VT.getVectorNumElements() + 03076 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 03077 <= N1.getValueType().getVectorNumElements()) 03078 && "Extract subvector overflow!"); 03079 } 03080 03081 // Trivial extraction. 03082 if (VT.getSimpleVT() == N1.getValueType().getSimpleVT()) 03083 return N1; 03084 } 03085 break; 03086 } 03087 } 03088 03089 // Perform trivial constant folding. 03090 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); 03091 if (SV.getNode()) return SV; 03092 03093 // Canonicalize constant to RHS if commutative. 03094 if (N1C && !N2C && isCommutativeBinOp(Opcode)) { 03095 std::swap(N1C, N2C); 03096 std::swap(N1, N2); 03097 } 03098 03099 // Constant fold FP operations. 03100 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 03101 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 03102 if (N1CFP) { 03103 if (!N2CFP && isCommutativeBinOp(Opcode)) { 03104 // Canonicalize constant to RHS if commutative. 03105 std::swap(N1CFP, N2CFP); 03106 std::swap(N1, N2); 03107 } else if (N2CFP) { 03108 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 03109 APFloat::opStatus s; 03110 switch (Opcode) { 03111 case ISD::FADD: 03112 s = V1.add(V2, APFloat::rmNearestTiesToEven); 03113 if (s != APFloat::opInvalidOp) 03114 return getConstantFP(V1, VT); 03115 break; 03116 case ISD::FSUB: 03117 s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 03118 if (s!=APFloat::opInvalidOp) 03119 return getConstantFP(V1, VT); 03120 break; 03121 case ISD::FMUL: 03122 s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 03123 if (s!=APFloat::opInvalidOp) 03124 return getConstantFP(V1, VT); 03125 break; 03126 case ISD::FDIV: 03127 s = V1.divide(V2, APFloat::rmNearestTiesToEven); 03128 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 03129 return getConstantFP(V1, VT); 03130 break; 03131 case ISD::FREM : 03132 s = V1.mod(V2, APFloat::rmNearestTiesToEven); 03133 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 03134 return getConstantFP(V1, VT); 03135 break; 03136 case ISD::FCOPYSIGN: 03137 V1.copySign(V2); 03138 return getConstantFP(V1, VT); 03139 default: break; 03140 } 03141 } 03142 03143 if (Opcode == ISD::FP_ROUND) { 03144 APFloat V = N1CFP->getValueAPF(); // make copy 03145 bool ignored; 03146 // This can return overflow, underflow, or inexact; we don't care. 03147 // FIXME need to be more flexible about rounding mode. 03148 (void)V.convert(EVTToAPFloatSemantics(VT), 03149 APFloat::rmNearestTiesToEven, &ignored); 03150 return getConstantFP(V, VT); 03151 } 03152 } 03153 03154 // Canonicalize an UNDEF to the RHS, even over a constant. 03155 if (N1.getOpcode() == ISD::UNDEF) { 03156 if (isCommutativeBinOp(Opcode)) { 03157 std::swap(N1, N2); 03158 } else { 03159 switch (Opcode) { 03160 case ISD::FP_ROUND_INREG: 03161 case ISD::SIGN_EXTEND_INREG: 03162 case ISD::SUB: 03163 case ISD::FSUB: 03164 case ISD::FDIV: 03165 case ISD::FREM: 03166 case ISD::SRA: 03167 return N1; // fold op(undef, arg2) -> undef 03168 case ISD::UDIV: 03169 case ISD::SDIV: 03170 case ISD::UREM: 03171 case ISD::SREM: 03172 case ISD::SRL: 03173 case ISD::SHL: 03174 if (!VT.isVector()) 03175 return getConstant(0, VT); // fold op(undef, arg2) -> 0 03176 // For vectors, we can't easily build an all zero vector, just return 03177 // the LHS. 03178 return N2; 03179 } 03180 } 03181 } 03182 03183 // Fold a bunch of operators when the RHS is undef. 03184 if (N2.getOpcode() == ISD::UNDEF) { 03185 switch (Opcode) { 03186 case ISD::XOR: 03187 if (N1.getOpcode() == ISD::UNDEF) 03188 // Handle undef ^ undef -> 0 special case. This is a common 03189 // idiom (misuse). 03190 return getConstant(0, VT); 03191 // fallthrough 03192 case ISD::ADD: 03193 case ISD::ADDC: 03194 case ISD::ADDE: 03195 case ISD::SUB: 03196 case ISD::UDIV: 03197 case ISD::SDIV: 03198 case ISD::UREM: 03199 case ISD::SREM: 03200 return N2; // fold op(arg1, undef) -> undef 03201 case ISD::FADD: 03202 case ISD::FSUB: 03203 case ISD::FMUL: 03204 case ISD::FDIV: 03205 case ISD::FREM: 03206 if (getTarget().Options.UnsafeFPMath) 03207 return N2; 03208 break; 03209 case ISD::MUL: 03210 case ISD::AND: 03211 case ISD::SRL: 03212 case ISD::SHL: 03213 if (!VT.isVector()) 03214 return getConstant(0, VT); // fold op(arg1, undef) -> 0 03215 // For vectors, we can't easily build an all zero vector, just return 03216 // the LHS. 03217 return N1; 03218 case ISD::OR: 03219 if (!VT.isVector()) 03220 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 03221 // For vectors, we can't easily build an all one vector, just return 03222 // the LHS. 03223 return N1; 03224 case ISD::SRA: 03225 return N1; 03226 } 03227 } 03228 03229 // Memoize this node if possible. 03230 SDNode *N; 03231 SDVTList VTs = getVTList(VT); 03232 if (VT != MVT::Glue) { 03233 SDValue Ops[] = { N1, N2 }; 03234 FoldingSetNodeID ID; 03235 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 03236 void *IP = 0; 03237 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 03238 return SDValue(E, 0); 03239 03240 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2); 03241 CSEMap.InsertNode(N, IP); 03242 } else { 03243 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2); 03244 } 03245 03246 AllNodes.push_back(N); 03247 #ifndef NDEBUG 03248 VerifySDNode(N); 03249 #endif 03250 return SDValue(N, 0); 03251 } 03252 03253 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 03254 SDValue N1, SDValue N2, SDValue N3) { 03255 // Perform various simplifications. 03256 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 03257 switch (Opcode) { 03258 case ISD::FMA: { 03259 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 03260 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 03261 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 03262 if (N1CFP && N2CFP && N3CFP) { 03263 APFloat V1 = N1CFP->getValueAPF(); 03264 const APFloat &V2 = N2CFP->getValueAPF(); 03265 const APFloat &V3 = N3CFP->getValueAPF(); 03266 APFloat::opStatus s = 03267 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 03268 if (s != APFloat::opInvalidOp) 03269 return getConstantFP(V1, VT); 03270 } 03271 break; 03272 } 03273 case ISD::CONCAT_VECTORS: 03274 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 03275 // one big BUILD_VECTOR. 03276 if (N1.getOpcode() == ISD::BUILD_VECTOR && 03277 N2.getOpcode() == ISD::BUILD_VECTOR && 03278 N3.getOpcode() == ISD::BUILD_VECTOR) { 03279 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 03280 N1.getNode()->op_end()); 03281 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 03282 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 03283 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 03284 } 03285 break; 03286 case ISD::SETCC: { 03287 // Use FoldSetCC to simplify SETCC's. 03288 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 03289 if (Simp.getNode()) return Simp; 03290 break; 03291 } 03292 case ISD::SELECT: 03293 if (N1C) { 03294 if (N1C->getZExtValue()) 03295 return N2; // select true, X, Y -> X 03296 return N3; // select false, X, Y -> Y 03297 } 03298 03299 if (N2 == N3) return N2; // select C, X, X -> X 03300 break; 03301 case ISD::VECTOR_SHUFFLE: 03302 llvm_unreachable("should use getVectorShuffle constructor!"); 03303 case ISD::INSERT_SUBVECTOR: { 03304 SDValue Index = N3; 03305 if (VT.isSimple() && N1.getValueType().isSimple() 03306 && N2.getValueType().isSimple()) { 03307 assert(VT.isVector() && N1.getValueType().isVector() && 03308 N2.getValueType().isVector() && 03309 "Insert subvector VTs must be a vectors"); 03310 assert(VT == N1.getValueType() && 03311 "Dest and insert subvector source types must match!"); 03312 assert(N2.getValueType().getSimpleVT() <= N1.getValueType().getSimpleVT() && 03313 "Insert subvector must be from smaller vector to larger vector!"); 03314 if (isa<ConstantSDNode>(Index.getNode())) { 03315 assert((N2.getValueType().getVectorNumElements() + 03316 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 03317 <= VT.getVectorNumElements()) 03318 && "Insert subvector overflow!"); 03319 } 03320 03321 // Trivial insertion. 03322 if (VT.getSimpleVT() == N2.getValueType().getSimpleVT()) 03323 return N2; 03324 } 03325 break; 03326 } 03327 case ISD::BITCAST: 03328 // Fold bit_convert nodes from a type to themselves. 03329 if (N1.getValueType() == VT) 03330 return N1; 03331 break; 03332 } 03333 03334 // Memoize node if it doesn't produce a flag. 03335 SDNode *N; 03336 SDVTList VTs = getVTList(VT); 03337 if (VT != MVT::Glue) { 03338 SDValue Ops[] = { N1, N2, N3 }; 03339 FoldingSetNodeID ID; 03340 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 03341 void *IP = 0; 03342 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 03343 return SDValue(E, 0); 03344 03345 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, N3); 03346 CSEMap.InsertNode(N, IP); 03347 } else { 03348 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, N3); 03349 } 03350 03351 AllNodes.push_back(N); 03352 #ifndef NDEBUG 03353 VerifySDNode(N); 03354 #endif 03355 return SDValue(N, 0); 03356 } 03357 03358 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 03359 SDValue N1, SDValue N2, SDValue N3, 03360 SDValue N4) { 03361 SDValue Ops[] = { N1, N2, N3, N4 }; 03362 return getNode(Opcode, DL, VT, Ops, 4); 03363 } 03364 03365 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 03366 SDValue N1, SDValue N2, SDValue N3, 03367 SDValue N4, SDValue N5) { 03368 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 03369 return getNode(Opcode, DL, VT, Ops, 5); 03370 } 03371 03372 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all 03373 /// the incoming stack arguments to be loaded from the stack. 03374 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 03375 SmallVector<SDValue, 8> ArgChains; 03376 03377 // Include the original chain at the beginning of the list. When this is 03378 // used by target LowerCall hooks, this helps legalize find the 03379 // CALLSEQ_BEGIN node. 03380 ArgChains.push_back(Chain); 03381 03382 // Add a chain value for each stack argument. 03383 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 03384 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 03385 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 03386 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 03387 if (FI->getIndex() < 0) 03388 ArgChains.push_back(SDValue(L, 1)); 03389 03390 // Build a tokenfactor for all the chains. 03391 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, 03392 &ArgChains[0], ArgChains.size()); 03393 } 03394 03395 /// getMemsetValue - Vectorized representation of the memset value 03396 /// operand. 03397 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 03398 SDLoc dl) { 03399 assert(Value.getOpcode() != ISD::UNDEF); 03400 03401 unsigned NumBits = VT.getScalarType().getSizeInBits(); 03402 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 03403 assert(C->getAPIntValue().getBitWidth() == 8); 03404 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 03405 if (VT.isInteger()) 03406 return DAG.getConstant(Val, VT); 03407 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); 03408 } 03409 03410 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 03411 if (NumBits > 8) { 03412 // Use a multiplication with 0x010101... to extend the input to the 03413 // required length. 03414 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 03415 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 03416 } 03417 03418 return Value; 03419 } 03420 03421 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only 03422 /// used when a memcpy is turned into a memset when the source is a constant 03423 /// string ptr. 03424 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG, 03425 const TargetLowering &TLI, StringRef Str) { 03426 // Handle vector with all elements zero. 03427 if (Str.empty()) { 03428 if (VT.isInteger()) 03429 return DAG.getConstant(0, VT); 03430 else if (VT == MVT::f32 || VT == MVT::f64) 03431 return DAG.getConstantFP(0.0, VT); 03432 else if (VT.isVector()) { 03433 unsigned NumElts = VT.getVectorNumElements(); 03434 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 03435 return DAG.getNode(ISD::BITCAST, dl, VT, 03436 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 03437 EltVT, NumElts))); 03438 } else 03439 llvm_unreachable("Expected type!"); 03440 } 03441 03442 assert(!VT.isVector() && "Can't handle vector type here!"); 03443 unsigned NumVTBits = VT.getSizeInBits(); 03444 unsigned NumVTBytes = NumVTBits / 8; 03445 unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); 03446 03447 APInt Val(NumVTBits, 0); 03448 if (TLI.isLittleEndian()) { 03449 for (unsigned i = 0; i != NumBytes; ++i) 03450 Val |= (uint64_t)(unsigned char)Str[i] << i*8; 03451 } else { 03452 for (unsigned i = 0; i != NumBytes; ++i) 03453 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; 03454 } 03455 03456 // If the "cost" of materializing the integer immediate is 1 or free, then 03457 // it is cost effective to turn the load into the immediate. 03458 const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); 03459 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) 03460 return DAG.getConstant(Val, VT); 03461 return SDValue(0, 0); 03462 } 03463 03464 /// getMemBasePlusOffset - Returns base and offset node for the 03465 /// 03466 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl, 03467 SelectionDAG &DAG) { 03468 EVT VT = Base.getValueType(); 03469 return DAG.getNode(ISD::ADD, dl, 03470 VT, Base, DAG.getConstant(Offset, VT)); 03471 } 03472 03473 /// isMemSrcFromString - Returns true if memcpy source is a string constant. 03474 /// 03475 static bool isMemSrcFromString(SDValue Src, StringRef &Str) { 03476 unsigned SrcDelta = 0; 03477 GlobalAddressSDNode *G = NULL; 03478 if (Src.getOpcode() == ISD::GlobalAddress) 03479 G = cast<GlobalAddressSDNode>(Src); 03480 else if (Src.getOpcode() == ISD::ADD && 03481 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 03482 Src.getOperand(1).getOpcode() == ISD::Constant) { 03483 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 03484 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 03485 } 03486 if (!G) 03487 return false; 03488 03489 return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false); 03490 } 03491 03492 /// FindOptimalMemOpLowering - Determines the optimial series memory ops 03493 /// to replace the memset / memcpy. Return true if the number of memory ops 03494 /// is below the threshold. It returns the types of the sequence of 03495 /// memory ops to perform memset / memcpy by reference. 03496 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 03497 unsigned Limit, uint64_t Size, 03498 unsigned DstAlign, unsigned SrcAlign, 03499 bool IsMemset, 03500 bool ZeroMemset, 03501 bool MemcpyStrSrc, 03502 bool AllowOverlap, 03503 SelectionDAG &DAG, 03504 const TargetLowering &TLI) { 03505 assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 03506 "Expecting memcpy / memset source to meet alignment requirement!"); 03507 // If 'SrcAlign' is zero, that means the memory operation does not need to 03508 // load the value, i.e. memset or memcpy from constant string. Otherwise, 03509 // it's the inferred alignment of the source. 'DstAlign', on the other hand, 03510 // is the specified alignment of the memory operation. If it is zero, that 03511 // means it's possible to change the alignment of the destination. 03512 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 03513 // not need to be loaded. 03514 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 03515 IsMemset, ZeroMemset, MemcpyStrSrc, 03516 DAG.getMachineFunction()); 03517 03518 if (VT == MVT::Other) { 03519 if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() || 03520 TLI.allowsUnalignedMemoryAccesses(VT)) { 03521 VT = TLI.getPointerTy(); 03522 } else { 03523 switch (DstAlign & 7) { 03524 case 0: VT = MVT::i64; break; 03525 case 4: VT = MVT::i32; break; 03526 case 2: VT = MVT::i16; break; 03527 default: VT = MVT::i8; break; 03528 } 03529 } 03530 03531 MVT LVT = MVT::i64; 03532 while (!TLI.isTypeLegal(LVT)) 03533 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 03534 assert(LVT.isInteger()); 03535 03536 if (VT.bitsGT(LVT)) 03537 VT = LVT; 03538 } 03539 03540 unsigned NumMemOps = 0; 03541 while (Size != 0) { 03542 unsigned VTSize = VT.getSizeInBits() / 8; 03543 while (VTSize > Size) { 03544 // For now, only use non-vector load / store's for the left-over pieces. 03545 EVT NewVT = VT; 03546 unsigned NewVTSize; 03547 03548 bool Found = false; 03549 if (VT.isVector() || VT.isFloatingPoint()) { 03550 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 03551 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 03552 TLI.isSafeMemOpType(NewVT.getSimpleVT())) 03553 Found = true; 03554 else if (NewVT == MVT::i64 && 03555 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 03556 TLI.isSafeMemOpType(MVT::f64)) { 03557 // i64 is usually not legal on 32-bit targets, but f64 may be. 03558 NewVT = MVT::f64; 03559 Found = true; 03560 } 03561 } 03562 03563 if (!Found) { 03564 do { 03565 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 03566 if (NewVT == MVT::i8) 03567 break; 03568 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 03569 } 03570 NewVTSize = NewVT.getSizeInBits() / 8; 03571 03572 // If the new VT cannot cover all of the remaining bits, then consider 03573 // issuing a (or a pair of) unaligned and overlapping load / store. 03574 // FIXME: Only does this for 64-bit or more since we don't have proper 03575 // cost model for unaligned load / store. 03576 bool Fast; 03577 if (NumMemOps && AllowOverlap && 03578 VTSize >= 8 && NewVTSize < Size && 03579 TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) 03580 VTSize = Size; 03581 else { 03582 VT = NewVT; 03583 VTSize = NewVTSize; 03584 } 03585 } 03586 03587 if (++NumMemOps > Limit) 03588 return false; 03589 03590 MemOps.push_back(VT); 03591 Size -= VTSize; 03592 } 03593 03594 return true; 03595 } 03596 03597 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 03598 SDValue Chain, SDValue Dst, 03599 SDValue Src, uint64_t Size, 03600 unsigned Align, bool isVol, 03601 bool AlwaysInline, 03602 MachinePointerInfo DstPtrInfo, 03603 MachinePointerInfo SrcPtrInfo) { 03604 // Turn a memcpy of undef to nop. 03605 if (Src.getOpcode() == ISD::UNDEF) 03606 return Chain; 03607 03608 // Expand memcpy to a series of load and store ops if the size operand falls 03609 // below a certain threshold. 03610 // TODO: In the AlwaysInline case, if the size is big then generate a loop 03611 // rather than maybe a humongous number of loads and stores. 03612 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 03613 std::vector<EVT> MemOps; 03614 bool DstAlignCanChange = false; 03615 MachineFunction &MF = DAG.getMachineFunction(); 03616 MachineFrameInfo *MFI = MF.getFrameInfo(); 03617 bool OptSize = 03618 MF.getFunction()->getAttributes(). 03619 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 03620 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 03621 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 03622 DstAlignCanChange = true; 03623 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 03624 if (Align > SrcAlign) 03625 SrcAlign = Align; 03626 StringRef Str; 03627 bool CopyFromStr = isMemSrcFromString(Src, Str); 03628 bool isZeroStr = CopyFromStr && Str.empty(); 03629 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 03630 03631 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 03632 (DstAlignCanChange ? 0 : Align), 03633 (isZeroStr ? 0 : SrcAlign), 03634 false, false, CopyFromStr, true, DAG, TLI)) 03635 return SDValue(); 03636 03637 if (DstAlignCanChange) { 03638 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 03639 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 03640 03641 // Don't promote to an alignment that would require dynamic stack 03642 // realignment. 03643 const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); 03644 if (!TRI->needsStackRealignment(MF)) 03645 while (NewAlign > Align && 03646 TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) 03647 NewAlign /= 2; 03648 03649 if (NewAlign > Align) { 03650 // Give the stack frame object a larger alignment if needed. 03651 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 03652 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 03653 Align = NewAlign; 03654 } 03655 } 03656 03657 SmallVector<SDValue, 8> OutChains; 03658 unsigned NumMemOps = MemOps.size(); 03659 uint64_t SrcOff = 0, DstOff = 0; 03660 for (unsigned i = 0; i != NumMemOps; ++i) { 03661 EVT VT = MemOps[i]; 03662 unsigned VTSize = VT.getSizeInBits() / 8; 03663 SDValue Value, Store; 03664 03665 if (VTSize > Size) { 03666 // Issuing an unaligned load / store pair that overlaps with the previous 03667 // pair. Adjust the offset accordingly. 03668 assert(i == NumMemOps-1 && i != 0); 03669 SrcOff -= VTSize - Size; 03670 DstOff -= VTSize - Size; 03671 } 03672 03673 if (CopyFromStr && 03674 (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 03675 // It's unlikely a store of a vector immediate can be done in a single 03676 // instruction. It would require a load from a constantpool first. 03677 // We only handle zero vectors here. 03678 // FIXME: Handle other cases where store of vector immediate is done in 03679 // a single instruction. 03680 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); 03681 if (Value.getNode()) 03682 Store = DAG.getStore(Chain, dl, Value, 03683 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 03684 DstPtrInfo.getWithOffset(DstOff), isVol, 03685 false, Align); 03686 } 03687 03688 if (!Store.getNode()) { 03689 // The type might not be legal for the target. This should only happen 03690 // if the type is smaller than a legal type, as on PPC, so the right 03691 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 03692 // to Load/Store if NVT==VT. 03693 // FIXME does the case above also need this? 03694 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 03695 assert(NVT.bitsGE(VT)); 03696 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 03697 getMemBasePlusOffset(Src, SrcOff, dl, DAG), 03698 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 03699 MinAlign(SrcAlign, SrcOff)); 03700 Store = DAG.getTruncStore(Chain, dl, Value, 03701 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 03702 DstPtrInfo.getWithOffset(DstOff), VT, isVol, 03703 false, Align); 03704 } 03705 OutChains.push_back(Store); 03706 SrcOff += VTSize; 03707 DstOff += VTSize; 03708 Size -= VTSize; 03709 } 03710 03711 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 03712 &OutChains[0], OutChains.size()); 03713 } 03714 03715 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 03716 SDValue Chain, SDValue Dst, 03717 SDValue Src, uint64_t Size, 03718 unsigned Align, bool isVol, 03719 bool AlwaysInline, 03720 MachinePointerInfo DstPtrInfo, 03721 MachinePointerInfo SrcPtrInfo) { 03722 // Turn a memmove of undef to nop. 03723 if (Src.getOpcode() == ISD::UNDEF) 03724 return Chain; 03725 03726 // Expand memmove to a series of load and store ops if the size operand falls 03727 // below a certain threshold. 03728 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 03729 std::vector<EVT> MemOps; 03730 bool DstAlignCanChange = false; 03731 MachineFunction &MF = DAG.getMachineFunction(); 03732 MachineFrameInfo *MFI = MF.getFrameInfo(); 03733 bool OptSize = MF.getFunction()->getAttributes(). 03734 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 03735 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 03736 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 03737 DstAlignCanChange = true; 03738 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 03739 if (Align > SrcAlign) 03740 SrcAlign = Align; 03741 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 03742 03743 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 03744 (DstAlignCanChange ? 0 : Align), SrcAlign, 03745 false, false, false, false, DAG, TLI)) 03746 return SDValue(); 03747 03748 if (DstAlignCanChange) { 03749 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 03750 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 03751 if (NewAlign > Align) { 03752 // Give the stack frame object a larger alignment if needed. 03753 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 03754 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 03755 Align = NewAlign; 03756 } 03757 } 03758 03759 uint64_t SrcOff = 0, DstOff = 0; 03760 SmallVector<SDValue, 8> LoadValues; 03761 SmallVector<SDValue, 8> LoadChains; 03762 SmallVector<SDValue, 8> OutChains; 03763 unsigned NumMemOps = MemOps.size(); 03764 for (unsigned i = 0; i < NumMemOps; i++) { 03765 EVT VT = MemOps[i]; 03766 unsigned VTSize = VT.getSizeInBits() / 8; 03767 SDValue Value, Store; 03768 03769 Value = DAG.getLoad(VT, dl, Chain, 03770 getMemBasePlusOffset(Src, SrcOff, dl, DAG), 03771 SrcPtrInfo.getWithOffset(SrcOff), isVol, 03772 false, false, SrcAlign); 03773 LoadValues.push_back(Value); 03774 LoadChains.push_back(Value.getValue(1)); 03775 SrcOff += VTSize; 03776 } 03777 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 03778 &LoadChains[0], LoadChains.size()); 03779 OutChains.clear(); 03780 for (unsigned i = 0; i < NumMemOps; i++) { 03781 EVT VT = MemOps[i]; 03782 unsigned VTSize = VT.getSizeInBits() / 8; 03783 SDValue Value, Store; 03784 03785 Store = DAG.getStore(Chain, dl, LoadValues[i], 03786 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 03787 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 03788 OutChains.push_back(Store); 03789 DstOff += VTSize; 03790 } 03791 03792 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 03793 &OutChains[0], OutChains.size()); 03794 } 03795 03796 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, 03797 SDValue Chain, SDValue Dst, 03798 SDValue Src, uint64_t Size, 03799 unsigned Align, bool isVol, 03800 MachinePointerInfo DstPtrInfo) { 03801 // Turn a memset of undef to nop. 03802 if (Src.getOpcode() == ISD::UNDEF) 03803 return Chain; 03804 03805 // Expand memset to a series of load/store ops if the size operand 03806 // falls below a certain threshold. 03807 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 03808 std::vector<EVT> MemOps; 03809 bool DstAlignCanChange = false; 03810 MachineFunction &MF = DAG.getMachineFunction(); 03811 MachineFrameInfo *MFI = MF.getFrameInfo(); 03812 bool OptSize = MF.getFunction()->getAttributes(). 03813 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 03814 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 03815 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 03816 DstAlignCanChange = true; 03817 bool IsZeroVal = 03818 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 03819 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 03820 Size, (DstAlignCanChange ? 0 : Align), 0, 03821 true, IsZeroVal, false, true, DAG, TLI)) 03822 return SDValue(); 03823 03824 if (DstAlignCanChange) { 03825 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 03826 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 03827 if (NewAlign > Align) { 03828 // Give the stack frame object a larger alignment if needed. 03829 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 03830 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 03831 Align = NewAlign; 03832 } 03833 } 03834 03835 SmallVector<SDValue, 8> OutChains; 03836 uint64_t DstOff = 0; 03837 unsigned NumMemOps = MemOps.size(); 03838 03839 // Find the largest store and generate the bit pattern for it. 03840 EVT LargestVT = MemOps[0]; 03841 for (unsigned i = 1; i < NumMemOps; i++) 03842 if (MemOps[i].bitsGT(LargestVT)) 03843 LargestVT = MemOps[i]; 03844 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 03845 03846 for (unsigned i = 0; i < NumMemOps; i++) { 03847 EVT VT = MemOps[i]; 03848 unsigned VTSize = VT.getSizeInBits() / 8; 03849 if (VTSize > Size) { 03850 // Issuing an unaligned load / store pair that overlaps with the previous 03851 // pair. Adjust the offset accordingly. 03852 assert(i == NumMemOps-1 && i != 0); 03853 DstOff -= VTSize - Size; 03854 } 03855 03856 // If this store is smaller than the largest store see whether we can get 03857 // the smaller value for free with a truncate. 03858 SDValue Value = MemSetValue; 03859 if (VT.bitsLT(LargestVT)) { 03860 if (!LargestVT.isVector() && !VT.isVector() && 03861 TLI.isTruncateFree(LargestVT, VT)) 03862 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 03863 else 03864 Value = getMemsetValue(Src, VT, DAG, dl); 03865 } 03866 assert(Value.getValueType() == VT && "Value with wrong type."); 03867 SDValue Store = DAG.getStore(Chain, dl, Value, 03868 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 03869 DstPtrInfo.getWithOffset(DstOff), 03870 isVol, false, Align); 03871 OutChains.push_back(Store); 03872 DstOff += VT.getSizeInBits() / 8; 03873 Size -= VTSize; 03874 } 03875 03876 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 03877 &OutChains[0], OutChains.size()); 03878 } 03879 03880 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, 03881 SDValue Src, SDValue Size, 03882 unsigned Align, bool isVol, bool AlwaysInline, 03883 MachinePointerInfo DstPtrInfo, 03884 MachinePointerInfo SrcPtrInfo) { 03885 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 03886 03887 // Check to see if we should lower the memcpy to loads and stores first. 03888 // For cases within the target-specified limits, this is the best choice. 03889 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 03890 if (ConstantSize) { 03891 // Memcpy with size zero? Just return the original chain. 03892 if (ConstantSize->isNullValue()) 03893 return Chain; 03894 03895 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 03896 ConstantSize->getZExtValue(),Align, 03897 isVol, false, DstPtrInfo, SrcPtrInfo); 03898 if (Result.getNode()) 03899 return Result; 03900 } 03901 03902 // Then check to see if we should lower the memcpy with target-specific 03903 // code. If the target chooses to do this, this is the next best. 03904 SDValue Result = 03905 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 03906 isVol, AlwaysInline, 03907 DstPtrInfo, SrcPtrInfo); 03908 if (Result.getNode()) 03909 return Result; 03910 03911 // If we really need inline code and the target declined to provide it, 03912 // use a (potentially long) sequence of loads and stores. 03913 if (AlwaysInline) { 03914 assert(ConstantSize && "AlwaysInline requires a constant size!"); 03915 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 03916 ConstantSize->getZExtValue(), Align, isVol, 03917 true, DstPtrInfo, SrcPtrInfo); 03918 } 03919 03920 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 03921 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 03922 // respect volatile, so they may do things like read or write memory 03923 // beyond the given memory regions. But fixing this isn't easy, and most 03924 // people don't care. 03925 03926 // Emit a library call. 03927 TargetLowering::ArgListTy Args; 03928 TargetLowering::ArgListEntry Entry; 03929 Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext()); 03930 Entry.Node = Dst; Args.push_back(Entry); 03931 Entry.Node = Src; Args.push_back(Entry); 03932 Entry.Node = Size; Args.push_back(Entry); 03933 // FIXME: pass in SDLoc 03934 TargetLowering:: 03935 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 03936 false, false, false, false, 0, 03937 TLI.getLibcallCallingConv(RTLIB::MEMCPY), 03938 /*isTailCall=*/false, 03939 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 03940 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY), 03941 TLI.getPointerTy()), 03942 Args, *this, dl); 03943 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI); 03944 03945 return CallResult.second; 03946 } 03947 03948 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, 03949 SDValue Src, SDValue Size, 03950 unsigned Align, bool isVol, 03951 MachinePointerInfo DstPtrInfo, 03952 MachinePointerInfo SrcPtrInfo) { 03953 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 03954 03955 // Check to see if we should lower the memmove to loads and stores first. 03956 // For cases within the target-specified limits, this is the best choice. 03957 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 03958 if (ConstantSize) { 03959 // Memmove with size zero? Just return the original chain. 03960 if (ConstantSize->isNullValue()) 03961 return Chain; 03962 03963 SDValue Result = 03964 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 03965 ConstantSize->getZExtValue(), Align, isVol, 03966 false, DstPtrInfo, SrcPtrInfo); 03967 if (Result.getNode()) 03968 return Result; 03969 } 03970 03971 // Then check to see if we should lower the memmove with target-specific 03972 // code. If the target chooses to do this, this is the next best. 03973 SDValue Result = 03974 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 03975 DstPtrInfo, SrcPtrInfo); 03976 if (Result.getNode()) 03977 return Result; 03978 03979 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 03980 // not be safe. See memcpy above for more details. 03981 03982 // Emit a library call. 03983 TargetLowering::ArgListTy Args; 03984 TargetLowering::ArgListEntry Entry; 03985 Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext()); 03986 Entry.Node = Dst; Args.push_back(Entry); 03987 Entry.Node = Src; Args.push_back(Entry); 03988 Entry.Node = Size; Args.push_back(Entry); 03989 // FIXME: pass in SDLoc 03990 TargetLowering:: 03991 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 03992 false, false, false, false, 0, 03993 TLI.getLibcallCallingConv(RTLIB::MEMMOVE), 03994 /*isTailCall=*/false, 03995 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 03996 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE), 03997 TLI.getPointerTy()), 03998 Args, *this, dl); 03999 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI); 04000 04001 return CallResult.second; 04002 } 04003 04004 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, 04005 SDValue Src, SDValue Size, 04006 unsigned Align, bool isVol, 04007 MachinePointerInfo DstPtrInfo) { 04008 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 04009 04010 // Check to see if we should lower the memset to stores first. 04011 // For cases within the target-specified limits, this is the best choice. 04012 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 04013 if (ConstantSize) { 04014 // Memset with size zero? Just return the original chain. 04015 if (ConstantSize->isNullValue()) 04016 return Chain; 04017 04018 SDValue Result = 04019 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 04020 Align, isVol, DstPtrInfo); 04021 04022 if (Result.getNode()) 04023 return Result; 04024 } 04025 04026 // Then check to see if we should lower the memset with target-specific 04027 // code. If the target chooses to do this, this is the next best. 04028 SDValue Result = 04029 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 04030 DstPtrInfo); 04031 if (Result.getNode()) 04032 return Result; 04033 04034 // Emit a library call. 04035 Type *IntPtrTy = TLI.getDataLayout()->getIntPtrType(*getContext()); 04036 TargetLowering::ArgListTy Args; 04037 TargetLowering::ArgListEntry Entry; 04038 Entry.Node = Dst; Entry.Ty = IntPtrTy; 04039 Args.push_back(Entry); 04040 // Extend or truncate the argument to be an i32 value for the call. 04041 if (Src.getValueType().bitsGT(MVT::i32)) 04042 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 04043 else 04044 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 04045 Entry.Node = Src; 04046 Entry.Ty = Type::getInt32Ty(*getContext()); 04047 Entry.isSExt = true; 04048 Args.push_back(Entry); 04049 Entry.Node = Size; 04050 Entry.Ty = IntPtrTy; 04051 Entry.isSExt = false; 04052 Args.push_back(Entry); 04053 // FIXME: pass in SDLoc 04054 TargetLowering:: 04055 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 04056 false, false, false, false, 0, 04057 TLI.getLibcallCallingConv(RTLIB::MEMSET), 04058 /*isTailCall=*/false, 04059 /*doesNotReturn*/false, /*isReturnValueUsed=*/false, 04060 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET), 04061 TLI.getPointerTy()), 04062 Args, *this, dl); 04063 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI); 04064 04065 return CallResult.second; 04066 } 04067 04068 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 04069 SDValue Chain, SDValue Ptr, SDValue Cmp, 04070 SDValue Swp, MachinePointerInfo PtrInfo, 04071 unsigned Alignment, 04072 AtomicOrdering Ordering, 04073 SynchronizationScope SynchScope) { 04074 if (Alignment == 0) // Ensure that codegen never sees alignment 0 04075 Alignment = getEVTAlignment(MemVT); 04076 04077 MachineFunction &MF = getMachineFunction(); 04078 04079 // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. 04080 // For now, atomics are considered to be volatile always. 04081 // FIXME: Volatile isn't really correct; we should keep track of atomic 04082 // orderings in the memoperand. 04083 unsigned Flags = MachineMemOperand::MOVolatile; 04084 if (Opcode != ISD::ATOMIC_STORE) 04085 Flags |= MachineMemOperand::MOLoad; 04086 if (Opcode != ISD::ATOMIC_LOAD) 04087 Flags |= MachineMemOperand::MOStore; 04088 04089 MachineMemOperand *MMO = 04090 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 04091 04092 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 04093 Ordering, SynchScope); 04094 } 04095 04096 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 04097 SDValue Chain, 04098 SDValue Ptr, SDValue Cmp, 04099 SDValue Swp, MachineMemOperand *MMO, 04100 AtomicOrdering Ordering, 04101 SynchronizationScope SynchScope) { 04102 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 04103 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 04104 04105 EVT VT = Cmp.getValueType(); 04106 04107 SDVTList VTs = getVTList(VT, MVT::Other); 04108 FoldingSetNodeID ID; 04109 ID.AddInteger(MemVT.getRawBits()); 04110 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 04111 AddNodeIDNode(ID, Opcode, VTs, Ops, 4); 04112 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 04113 void* IP = 0; 04114 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 04115 cast<AtomicSDNode>(E)->refineAlignment(MMO); 04116 return SDValue(E, 0); 04117 } 04118 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, MemVT, Chain, 04119 Ptr, Cmp, Swp, MMO, Ordering, 04120 SynchScope); 04121 CSEMap.InsertNode(N, IP); 04122 AllNodes.push_back(N); 04123 return SDValue(N, 0); 04124 } 04125 04126 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 04127 SDValue Chain, 04128 SDValue Ptr, SDValue Val, 04129 const Value* PtrVal, 04130 unsigned Alignment, 04131 AtomicOrdering Ordering, 04132 SynchronizationScope SynchScope) { 04133 if (Alignment == 0) // Ensure that codegen never sees alignment 0 04134 Alignment = getEVTAlignment(MemVT); 04135 04136 MachineFunction &MF = getMachineFunction(); 04137 // An atomic store does not load. An atomic load does not store. 04138 // (An atomicrmw obviously both loads and stores.) 04139 // For now, atomics are considered to be volatile always, and they are 04140 // chained as such. 04141 // FIXME: Volatile isn't really correct; we should keep track of atomic 04142 // orderings in the memoperand. 04143 unsigned Flags = MachineMemOperand::MOVolatile; 04144 if (Opcode != ISD::ATOMIC_STORE) 04145 Flags |= MachineMemOperand::MOLoad; 04146 if (Opcode != ISD::ATOMIC_LOAD) 04147 Flags |= MachineMemOperand::MOStore; 04148 04149 MachineMemOperand *MMO = 04150 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 04151 MemVT.getStoreSize(), Alignment); 04152 04153 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 04154 Ordering, SynchScope); 04155 } 04156 04157 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 04158 SDValue Chain, 04159 SDValue Ptr, SDValue Val, 04160 MachineMemOperand *MMO, 04161 AtomicOrdering Ordering, 04162 SynchronizationScope SynchScope) { 04163 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 04164 Opcode == ISD::ATOMIC_LOAD_SUB || 04165 Opcode == ISD::ATOMIC_LOAD_AND || 04166 Opcode == ISD::ATOMIC_LOAD_OR || 04167 Opcode == ISD::ATOMIC_LOAD_XOR || 04168 Opcode == ISD::ATOMIC_LOAD_NAND || 04169 Opcode == ISD::ATOMIC_LOAD_MIN || 04170 Opcode == ISD::ATOMIC_LOAD_MAX || 04171 Opcode == ISD::ATOMIC_LOAD_UMIN || 04172 Opcode == ISD::ATOMIC_LOAD_UMAX || 04173 Opcode == ISD::ATOMIC_SWAP || 04174 Opcode == ISD::ATOMIC_STORE) && 04175 "Invalid Atomic Op"); 04176 04177 EVT VT = Val.getValueType(); 04178 04179 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 04180 getVTList(VT, MVT::Other); 04181 FoldingSetNodeID ID; 04182 ID.AddInteger(MemVT.getRawBits()); 04183 SDValue Ops[] = {Chain, Ptr, Val}; 04184 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 04185 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 04186 void* IP = 0; 04187 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 04188 cast<AtomicSDNode>(E)->refineAlignment(MMO); 04189 return SDValue(E, 0); 04190 } 04191 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, MemVT, Chain, 04192 Ptr, Val, MMO, 04193 Ordering, SynchScope); 04194 CSEMap.InsertNode(N, IP); 04195 AllNodes.push_back(N); 04196 return SDValue(N, 0); 04197 } 04198 04199 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 04200 EVT VT, SDValue Chain, 04201 SDValue Ptr, 04202 const Value* PtrVal, 04203 unsigned Alignment, 04204 AtomicOrdering Ordering, 04205 SynchronizationScope SynchScope) { 04206 if (Alignment == 0) // Ensure that codegen never sees alignment 0 04207 Alignment = getEVTAlignment(MemVT); 04208 04209 MachineFunction &MF = getMachineFunction(); 04210 // An atomic store does not load. An atomic load does not store. 04211 // (An atomicrmw obviously both loads and stores.) 04212 // For now, atomics are considered to be volatile always, and they are 04213 // chained as such. 04214 // FIXME: Volatile isn't really correct; we should keep track of atomic 04215 // orderings in the memoperand. 04216 unsigned Flags = MachineMemOperand::MOVolatile; 04217 if (Opcode != ISD::ATOMIC_STORE) 04218 Flags |= MachineMemOperand::MOLoad; 04219 if (Opcode != ISD::ATOMIC_LOAD) 04220 Flags |= MachineMemOperand::MOStore; 04221 04222 MachineMemOperand *MMO = 04223 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 04224 MemVT.getStoreSize(), Alignment); 04225 04226 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 04227 Ordering, SynchScope); 04228 } 04229 04230 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 04231 EVT VT, SDValue Chain, 04232 SDValue Ptr, 04233 MachineMemOperand *MMO, 04234 AtomicOrdering Ordering, 04235 SynchronizationScope SynchScope) { 04236 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 04237 04238 SDVTList VTs = getVTList(VT, MVT::Other); 04239 FoldingSetNodeID ID; 04240 ID.AddInteger(MemVT.getRawBits()); 04241 SDValue Ops[] = {Chain, Ptr}; 04242 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 04243 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 04244 void* IP = 0; 04245 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 04246 cast<AtomicSDNode>(E)->refineAlignment(MMO); 04247 return SDValue(E, 0); 04248 } 04249 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, MemVT, Chain, 04250 Ptr, MMO, Ordering, SynchScope); 04251 CSEMap.InsertNode(N, IP); 04252 AllNodes.push_back(N); 04253 return SDValue(N, 0); 04254 } 04255 04256 /// getMergeValues - Create a MERGE_VALUES node from the given operands. 04257 SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 04258 SDLoc dl) { 04259 if (NumOps == 1) 04260 return Ops[0]; 04261 04262 SmallVector<EVT, 4> VTs; 04263 VTs.reserve(NumOps); 04264 for (unsigned i = 0; i < NumOps; ++i) 04265 VTs.push_back(Ops[i].getValueType()); 04266 return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 04267 Ops, NumOps); 04268 } 04269 04270 SDValue 04271 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, 04272 const EVT *VTs, unsigned NumVTs, 04273 const SDValue *Ops, unsigned NumOps, 04274 EVT MemVT, MachinePointerInfo PtrInfo, 04275 unsigned Align, bool Vol, 04276 bool ReadMem, bool WriteMem) { 04277 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 04278 MemVT, PtrInfo, Align, Vol, 04279 ReadMem, WriteMem); 04280 } 04281 04282 SDValue 04283 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 04284 const SDValue *Ops, unsigned NumOps, 04285 EVT MemVT, MachinePointerInfo PtrInfo, 04286 unsigned Align, bool Vol, 04287 bool ReadMem, bool WriteMem) { 04288 if (Align == 0) // Ensure that codegen never sees alignment 0 04289 Align = getEVTAlignment(MemVT); 04290 04291 MachineFunction &MF = getMachineFunction(); 04292 unsigned Flags = 0; 04293 if (WriteMem) 04294 Flags |= MachineMemOperand::MOStore; 04295 if (ReadMem) 04296 Flags |= MachineMemOperand::MOLoad; 04297 if (Vol) 04298 Flags |= MachineMemOperand::MOVolatile; 04299 MachineMemOperand *MMO = 04300 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 04301 04302 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 04303 } 04304 04305 SDValue 04306 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 04307 const SDValue *Ops, unsigned NumOps, 04308 EVT MemVT, MachineMemOperand *MMO) { 04309 assert((Opcode == ISD::INTRINSIC_VOID || 04310 Opcode == ISD::INTRINSIC_W_CHAIN || 04311 Opcode == ISD::PREFETCH || 04312 Opcode == ISD::LIFETIME_START || 04313 Opcode == ISD::LIFETIME_END || 04314 (Opcode <= INT_MAX && 04315 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 04316 "Opcode is not a memory-accessing opcode!"); 04317 04318 // Memoize the node unless it returns a flag. 04319 MemIntrinsicSDNode *N; 04320 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 04321 FoldingSetNodeID ID; 04322 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 04323 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 04324 void *IP = 0; 04325 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 04326 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 04327 return SDValue(E, 0); 04328 } 04329 04330 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, Ops, NumOps, 04331 MemVT, MMO); 04332 CSEMap.InsertNode(N, IP); 04333 } else { 04334 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, Ops, NumOps, 04335 MemVT, MMO); 04336 } 04337 AllNodes.push_back(N); 04338 return SDValue(N, 0); 04339 } 04340 04341 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 04342 /// MachinePointerInfo record from it. This is particularly useful because the 04343 /// code generator has many cases where it doesn't bother passing in a 04344 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 04345 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 04346 // If this is FI+Offset, we can model it. 04347 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 04348 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 04349 04350 // If this is (FI+Offset1)+Offset2, we can model it. 04351 if (Ptr.getOpcode() != ISD::ADD || 04352 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 04353 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 04354 return MachinePointerInfo(); 04355 04356 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 04357 return MachinePointerInfo::getFixedStack(FI, Offset+ 04358 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 04359 } 04360 04361 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 04362 /// MachinePointerInfo record from it. This is particularly useful because the 04363 /// code generator has many cases where it doesn't bother passing in a 04364 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 04365 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 04366 // If the 'Offset' value isn't a constant, we can't handle this. 04367 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 04368 return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 04369 if (OffsetOp.getOpcode() == ISD::UNDEF) 04370 return InferPointerInfo(Ptr); 04371 return MachinePointerInfo(); 04372 } 04373 04374 04375 SDValue 04376 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 04377 EVT VT, SDLoc dl, SDValue Chain, 04378 SDValue Ptr, SDValue Offset, 04379 MachinePointerInfo PtrInfo, EVT MemVT, 04380 bool isVolatile, bool isNonTemporal, bool isInvariant, 04381 unsigned Alignment, const MDNode *TBAAInfo, 04382 const MDNode *Ranges) { 04383 assert(Chain.getValueType() == MVT::Other && 04384 "Invalid chain type"); 04385 if (Alignment == 0) // Ensure that codegen never sees alignment 0 04386 Alignment = getEVTAlignment(VT); 04387 04388 unsigned Flags = MachineMemOperand::MOLoad; 04389 if (isVolatile) 04390 Flags |= MachineMemOperand::MOVolatile; 04391 if (isNonTemporal) 04392 Flags |= MachineMemOperand::MONonTemporal; 04393 if (isInvariant) 04394 Flags |= MachineMemOperand::MOInvariant; 04395 04396 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 04397 // clients. 04398 if (PtrInfo.V == 0) 04399 PtrInfo = InferPointerInfo(Ptr, Offset); 04400 04401 MachineFunction &MF = getMachineFunction(); 04402 MachineMemOperand *MMO = 04403 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 04404 TBAAInfo, Ranges); 04405 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 04406 } 04407 04408 SDValue 04409 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 04410 EVT VT, SDLoc dl, SDValue Chain, 04411 SDValue Ptr, SDValue Offset, EVT MemVT, 04412 MachineMemOperand *MMO) { 04413 if (VT == MemVT) { 04414 ExtType = ISD::NON_EXTLOAD; 04415 } else if (ExtType == ISD::NON_EXTLOAD) { 04416 assert(VT == MemVT && "Non-extending load from different memory type!"); 04417 } else { 04418 // Extending load. 04419 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 04420 "Should only be an extending load, not truncating!"); 04421 assert(VT.isInteger() == MemVT.isInteger() && 04422 "Cannot convert from FP to Int or Int -> FP!"); 04423 assert(VT.isVector() == MemVT.isVector() && 04424 "Cannot use trunc store to convert to or from a vector!"); 04425 assert((!VT.isVector() || 04426 VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 04427 "Cannot use trunc store to change the number of vector elements!"); 04428 } 04429 04430 bool Indexed = AM != ISD::UNINDEXED; 04431 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 04432 "Unindexed load with an offset!"); 04433 04434 SDVTList VTs = Indexed ? 04435 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 04436 SDValue Ops[] = { Chain, Ptr, Offset }; 04437 FoldingSetNodeID ID; 04438 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 04439 ID.AddInteger(MemVT.getRawBits()); 04440 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 04441 MMO->isNonTemporal(), 04442 MMO->isInvariant())); 04443 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 04444 void *IP = 0; 04445 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 04446 cast<LoadSDNode>(E)->refineAlignment(MMO); 04447 return SDValue(E, 0); 04448 } 04449 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ExtType, 04450 MemVT, MMO); 04451 CSEMap.InsertNode(N, IP); 04452 AllNodes.push_back(N); 04453 return SDValue(N, 0); 04454 } 04455 04456 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 04457 SDValue Chain, SDValue Ptr, 04458 MachinePointerInfo PtrInfo, 04459 bool isVolatile, bool isNonTemporal, 04460 bool isInvariant, unsigned Alignment, 04461 const MDNode *TBAAInfo, 04462 const MDNode *Ranges) { 04463 SDValue Undef = getUNDEF(Ptr.getValueType()); 04464 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 04465 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 04466 TBAAInfo, Ranges); 04467 } 04468 04469 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 04470 SDValue Chain, SDValue Ptr, 04471 MachinePointerInfo PtrInfo, EVT MemVT, 04472 bool isVolatile, bool isNonTemporal, 04473 unsigned Alignment, const MDNode *TBAAInfo) { 04474 SDValue Undef = getUNDEF(Ptr.getValueType()); 04475 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 04476 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 04477 TBAAInfo); 04478 } 04479 04480 04481 SDValue 04482 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base, 04483 SDValue Offset, ISD::MemIndexedMode AM) { 04484 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 04485 assert(LD->getOffset().getOpcode() == ISD::UNDEF && 04486 "Load is already a indexed load!"); 04487 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 04488 LD->getChain(), Base, Offset, LD->getPointerInfo(), 04489 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 04490 false, LD->getAlignment()); 04491 } 04492 04493 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 04494 SDValue Ptr, MachinePointerInfo PtrInfo, 04495 bool isVolatile, bool isNonTemporal, 04496 unsigned Alignment, const MDNode *TBAAInfo) { 04497 assert(Chain.getValueType() == MVT::Other && 04498 "Invalid chain type"); 04499 if (Alignment == 0) // Ensure that codegen never sees alignment 0 04500 Alignment = getEVTAlignment(Val.getValueType()); 04501 04502 unsigned Flags = MachineMemOperand::MOStore; 04503 if (isVolatile) 04504 Flags |= MachineMemOperand::MOVolatile; 04505 if (isNonTemporal) 04506 Flags |= MachineMemOperand::MONonTemporal; 04507 04508 if (PtrInfo.V == 0) 04509 PtrInfo = InferPointerInfo(Ptr); 04510 04511 MachineFunction &MF = getMachineFunction(); 04512 MachineMemOperand *MMO = 04513 MF.getMachineMemOperand(PtrInfo, Flags, 04514 Val.getValueType().getStoreSize(), Alignment, 04515 TBAAInfo); 04516 04517 return getStore(Chain, dl, Val, Ptr, MMO); 04518 } 04519 04520 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 04521 SDValue Ptr, MachineMemOperand *MMO) { 04522 assert(Chain.getValueType() == MVT::Other && 04523 "Invalid chain type"); 04524 EVT VT = Val.getValueType(); 04525 SDVTList VTs = getVTList(MVT::Other); 04526 SDValue Undef = getUNDEF(Ptr.getValueType()); 04527 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 04528 FoldingSetNodeID ID; 04529 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 04530 ID.AddInteger(VT.getRawBits()); 04531 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 04532 MMO->isNonTemporal(), MMO->isInvariant())); 04533 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 04534 void *IP = 0; 04535 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 04536 cast<StoreSDNode>(E)->refineAlignment(MMO); 04537 return SDValue(E, 0); 04538 } 04539 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED, 04540 false, VT, MMO); 04541 CSEMap.InsertNode(N, IP); 04542 AllNodes.push_back(N); 04543 return SDValue(N, 0); 04544 } 04545 04546 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 04547 SDValue Ptr, MachinePointerInfo PtrInfo, 04548 EVT SVT,bool isVolatile, bool isNonTemporal, 04549 unsigned Alignment, 04550 const MDNode *TBAAInfo) { 04551 assert(Chain.getValueType() == MVT::Other && 04552 "Invalid chain type"); 04553 if (Alignment == 0) // Ensure that codegen never sees alignment 0 04554 Alignment = getEVTAlignment(SVT); 04555 04556 unsigned Flags = MachineMemOperand::MOStore; 04557 if (isVolatile) 04558 Flags |= MachineMemOperand::MOVolatile; 04559 if (isNonTemporal) 04560 Flags |= MachineMemOperand::MONonTemporal; 04561 04562 if (PtrInfo.V == 0) 04563 PtrInfo = InferPointerInfo(Ptr); 04564 04565 MachineFunction &MF = getMachineFunction(); 04566 MachineMemOperand *MMO = 04567 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 04568 TBAAInfo); 04569 04570 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 04571 } 04572 04573 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 04574 SDValue Ptr, EVT SVT, 04575 MachineMemOperand *MMO) { 04576 EVT VT = Val.getValueType(); 04577 04578 assert(Chain.getValueType() == MVT::Other && 04579 "Invalid chain type"); 04580 if (VT == SVT) 04581 return getStore(Chain, dl, Val, Ptr, MMO); 04582 04583 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 04584 "Should only be a truncating store, not extending!"); 04585 assert(VT.isInteger() == SVT.isInteger() && 04586 "Can't do FP-INT conversion!"); 04587 assert(VT.isVector() == SVT.isVector() && 04588 "Cannot use trunc store to convert to or from a vector!"); 04589 assert((!VT.isVector() || 04590 VT.getVectorNumElements() == SVT.getVectorNumElements()) && 04591 "Cannot use trunc store to change the number of vector elements!"); 04592 04593 SDVTList VTs = getVTList(MVT::Other); 04594 SDValue Undef = getUNDEF(Ptr.getValueType()); 04595 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 04596 FoldingSetNodeID ID; 04597 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 04598 ID.AddInteger(SVT.getRawBits()); 04599 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 04600 MMO->isNonTemporal(), MMO->isInvariant())); 04601 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 04602 void *IP = 0; 04603 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 04604 cast<StoreSDNode>(E)->refineAlignment(MMO); 04605 return SDValue(E, 0); 04606 } 04607 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED, 04608 true, SVT, MMO); 04609 CSEMap.InsertNode(N, IP); 04610 AllNodes.push_back(N); 04611 return SDValue(N, 0); 04612 } 04613 04614 SDValue 04615 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, 04616 SDValue Offset, ISD::MemIndexedMode AM) { 04617 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 04618 assert(ST->getOffset().getOpcode() == ISD::UNDEF && 04619 "Store is already a indexed store!"); 04620 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 04621 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 04622 FoldingSetNodeID ID; 04623 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 04624 ID.AddInteger(ST->getMemoryVT().getRawBits()); 04625 ID.AddInteger(ST->getRawSubclassData()); 04626 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 04627 void *IP = 0; 04628 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 04629 return SDValue(E, 0); 04630 04631 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 04632 ST->isTruncatingStore(), 04633 ST->getMemoryVT(), 04634 ST->getMemOperand()); 04635 CSEMap.InsertNode(N, IP); 04636 AllNodes.push_back(N); 04637 return SDValue(N, 0); 04638 } 04639 04640 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl, 04641 SDValue Chain, SDValue Ptr, 04642 SDValue SV, 04643 unsigned Align) { 04644 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 04645 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 04646 } 04647 04648 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 04649 const SDUse *Ops, unsigned NumOps) { 04650 switch (NumOps) { 04651 case 0: return getNode(Opcode, DL, VT); 04652 case 1: return getNode(Opcode, DL, VT, Ops[0]); 04653 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 04654 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 04655 default: break; 04656 } 04657 04658 // Copy from an SDUse array into an SDValue array for use with 04659 // the regular getNode logic. 04660 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 04661 return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 04662 } 04663 04664 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 04665 const SDValue *Ops, unsigned NumOps) { 04666 switch (NumOps) { 04667 case 0: return getNode(Opcode, DL, VT); 04668 case 1: return getNode(Opcode, DL, VT, Ops[0]); 04669 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 04670 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 04671 default: break; 04672 } 04673 04674 switch (Opcode) { 04675 default: break; 04676 case ISD::SELECT_CC: { 04677 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 04678 assert(Ops[0].getValueType() == Ops[1].getValueType() && 04679 "LHS and RHS of condition must have same type!"); 04680 assert(Ops[2].getValueType() == Ops[3].getValueType() && 04681 "True and False arms of SelectCC must have same type!"); 04682 assert(Ops[2].getValueType() == VT && 04683 "select_cc node must be of same type as true and false value!"); 04684 break; 04685 } 04686 case ISD::BR_CC: { 04687 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 04688 assert(Ops[2].getValueType() == Ops[3].getValueType() && 04689 "LHS/RHS of comparison should match types!"); 04690 break; 04691 } 04692 } 04693 04694 // Memoize nodes. 04695 SDNode *N; 04696 SDVTList VTs = getVTList(VT); 04697 04698 if (VT != MVT::Glue) { 04699 FoldingSetNodeID ID; 04700 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 04701 void *IP = 0; 04702 04703 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 04704 return SDValue(E, 0); 04705 04706 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops, NumOps); 04707 CSEMap.InsertNode(N, IP); 04708 } else { 04709 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops, NumOps); 04710 } 04711 04712 AllNodes.push_back(N); 04713 #ifndef NDEBUG 04714 VerifySDNode(N); 04715 #endif 04716 return SDValue(N, 0); 04717 } 04718 04719 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 04720 ArrayRef<EVT> ResultTys, 04721 const SDValue *Ops, unsigned NumOps) { 04722 return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 04723 Ops, NumOps); 04724 } 04725 04726 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 04727 const EVT *VTs, unsigned NumVTs, 04728 const SDValue *Ops, unsigned NumOps) { 04729 if (NumVTs == 1) 04730 return getNode(Opcode, DL, VTs[0], Ops, NumOps); 04731 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 04732 } 04733 04734 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 04735 const SDValue *Ops, unsigned NumOps) { 04736 if (VTList.NumVTs == 1) 04737 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 04738 04739 #if 0 04740 switch (Opcode) { 04741 // FIXME: figure out how to safely handle things like 04742 // int foo(int x) { return 1 << (x & 255); } 04743 // int bar() { return foo(256); } 04744 case ISD::SRA_PARTS: 04745 case ISD::SRL_PARTS: 04746 case ISD::SHL_PARTS: 04747 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 04748 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 04749 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 04750 else if (N3.getOpcode() == ISD::AND) 04751 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 04752 // If the and is only masking out bits that cannot effect the shift, 04753 // eliminate the and. 04754 unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 04755 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 04756 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 04757 } 04758 break; 04759 } 04760 #endif 04761 04762 // Memoize the node unless it returns a flag. 04763 SDNode *N; 04764 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 04765 FoldingSetNodeID ID; 04766 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 04767 void *IP = 0; 04768 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 04769 return SDValue(E, 0); 04770 04771 if (NumOps == 1) { 04772 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0]); 04773 } else if (NumOps == 2) { 04774 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1]); 04775 } else if (NumOps == 3) { 04776 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1], 04777 Ops[2]); 04778 } else { 04779 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops, NumOps); 04780 } 04781 CSEMap.InsertNode(N, IP); 04782 } else { 04783 if (NumOps == 1) { 04784 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0]); 04785 } else if (NumOps == 2) { 04786 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1]); 04787 } else if (NumOps == 3) { 04788 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1], 04789 Ops[2]); 04790 } else { 04791 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops, NumOps); 04792 } 04793 } 04794 AllNodes.push_back(N); 04795 #ifndef NDEBUG 04796 VerifySDNode(N); 04797 #endif 04798 return SDValue(N, 0); 04799 } 04800 04801 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) { 04802 return getNode(Opcode, DL, VTList, 0, 0); 04803 } 04804 04805 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 04806 SDValue N1) { 04807 SDValue Ops[] = { N1 }; 04808 return getNode(Opcode, DL, VTList, Ops, 1); 04809 } 04810 04811 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 04812 SDValue N1, SDValue N2) { 04813 SDValue Ops[] = { N1, N2 }; 04814 return getNode(Opcode, DL, VTList, Ops, 2); 04815 } 04816 04817 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 04818 SDValue N1, SDValue N2, SDValue N3) { 04819 SDValue Ops[] = { N1, N2, N3 }; 04820 return getNode(Opcode, DL, VTList, Ops, 3); 04821 } 04822 04823 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 04824 SDValue N1, SDValue N2, SDValue N3, 04825 SDValue N4) { 04826 SDValue Ops[] = { N1, N2, N3, N4 }; 04827 return getNode(Opcode, DL, VTList, Ops, 4); 04828 } 04829 04830 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 04831 SDValue N1, SDValue N2, SDValue N3, 04832 SDValue N4, SDValue N5) { 04833 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 04834 return getNode(Opcode, DL, VTList, Ops, 5); 04835 } 04836 04837 SDVTList SelectionDAG::getVTList(EVT VT) { 04838 return makeVTList(SDNode::getValueTypeList(VT), 1); 04839 } 04840 04841 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 04842 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 04843 E = VTList.rend(); I != E; ++I) 04844 if (I->NumVTs == 2 && I->VTs[0] == VT1 && I->VTs[1] == VT2) 04845 return *I; 04846 04847 EVT *Array = Allocator.Allocate<EVT>(2); 04848 Array[0] = VT1; 04849 Array[1] = VT2; 04850 SDVTList Result = makeVTList(Array, 2); 04851 VTList.push_back(Result); 04852 return Result; 04853 } 04854 04855 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 04856 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 04857 E = VTList.rend(); I != E; ++I) 04858 if (I->NumVTs == 3 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 04859 I->VTs[2] == VT3) 04860 return *I; 04861 04862 EVT *Array = Allocator.Allocate<EVT>(3); 04863 Array[0] = VT1; 04864 Array[1] = VT2; 04865 Array[2] = VT3; 04866 SDVTList Result = makeVTList(Array, 3); 04867 VTList.push_back(Result); 04868 return Result; 04869 } 04870 04871 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 04872 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 04873 E = VTList.rend(); I != E; ++I) 04874 if (I->NumVTs == 4 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 04875 I->VTs[2] == VT3 && I->VTs[3] == VT4) 04876 return *I; 04877 04878 EVT *Array = Allocator.Allocate<EVT>(4); 04879 Array[0] = VT1; 04880 Array[1] = VT2; 04881 Array[2] = VT3; 04882 Array[3] = VT4; 04883 SDVTList Result = makeVTList(Array, 4); 04884 VTList.push_back(Result); 04885 return Result; 04886 } 04887 04888 SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 04889 switch (NumVTs) { 04890 case 0: llvm_unreachable("Cannot have nodes without results!"); 04891 case 1: return getVTList(VTs[0]); 04892 case 2: return getVTList(VTs[0], VTs[1]); 04893 case 3: return getVTList(VTs[0], VTs[1], VTs[2]); 04894 case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]); 04895 default: break; 04896 } 04897 04898 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 04899 E = VTList.rend(); I != E; ++I) { 04900 if (I->NumVTs != NumVTs || VTs[0] != I->VTs[0] || VTs[1] != I->VTs[1]) 04901 continue; 04902 04903 if (std::equal(&VTs[2], &VTs[NumVTs], &I->VTs[2])) 04904 return *I; 04905 } 04906 04907 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 04908 std::copy(VTs, VTs+NumVTs, Array); 04909 SDVTList Result = makeVTList(Array, NumVTs); 04910 VTList.push_back(Result); 04911 return Result; 04912 } 04913 04914 04915 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the 04916 /// specified operands. If the resultant node already exists in the DAG, 04917 /// this does not modify the specified node, instead it returns the node that 04918 /// already exists. If the resultant node does not exist in the DAG, the 04919 /// input node is returned. As a degenerate case, if you specify the same 04920 /// input operands as the node already has, the input node is returned. 04921 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 04922 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 04923 04924 // Check to see if there is no change. 04925 if (Op == N->getOperand(0)) return N; 04926 04927 // See if the modified node already exists. 04928 void *InsertPos = 0; 04929 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 04930 return Existing; 04931 04932 // Nope it doesn't. Remove the node from its current place in the maps. 04933 if (InsertPos) 04934 if (!RemoveNodeFromCSEMaps(N)) 04935 InsertPos = 0; 04936 04937 // Now we update the operands. 04938 N->OperandList[0].set(Op); 04939 04940 // If this gets put into a CSE map, add it. 04941 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 04942 return N; 04943 } 04944 04945 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 04946 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 04947 04948 // Check to see if there is no change. 04949 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 04950 return N; // No operands changed, just return the input node. 04951 04952 // See if the modified node already exists. 04953 void *InsertPos = 0; 04954 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 04955 return Existing; 04956 04957 // Nope it doesn't. Remove the node from its current place in the maps. 04958 if (InsertPos) 04959 if (!RemoveNodeFromCSEMaps(N)) 04960 InsertPos = 0; 04961 04962 // Now we update the operands. 04963 if (N->OperandList[0] != Op1) 04964 N->OperandList[0].set(Op1); 04965 if (N->OperandList[1] != Op2) 04966 N->OperandList[1].set(Op2); 04967 04968 // If this gets put into a CSE map, add it. 04969 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 04970 return N; 04971 } 04972 04973 SDNode *SelectionDAG:: 04974 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 04975 SDValue Ops[] = { Op1, Op2, Op3 }; 04976 return UpdateNodeOperands(N, Ops, 3); 04977 } 04978 04979 SDNode *SelectionDAG:: 04980 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 04981 SDValue Op3, SDValue Op4) { 04982 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 04983 return UpdateNodeOperands(N, Ops, 4); 04984 } 04985 04986 SDNode *SelectionDAG:: 04987 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 04988 SDValue Op3, SDValue Op4, SDValue Op5) { 04989 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 04990 return UpdateNodeOperands(N, Ops, 5); 04991 } 04992 04993 SDNode *SelectionDAG:: 04994 UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 04995 assert(N->getNumOperands() == NumOps && 04996 "Update with wrong number of operands"); 04997 04998 // Check to see if there is no change. 04999 bool AnyChange = false; 05000 for (unsigned i = 0; i != NumOps; ++i) { 05001 if (Ops[i] != N->getOperand(i)) { 05002 AnyChange = true; 05003 break; 05004 } 05005 } 05006 05007 // No operands changed, just return the input node. 05008 if (!AnyChange) return N; 05009 05010 // See if the modified node already exists. 05011 void *InsertPos = 0; 05012 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 05013 return Existing; 05014 05015 // Nope it doesn't. Remove the node from its current place in the maps. 05016 if (InsertPos) 05017 if (!RemoveNodeFromCSEMaps(N)) 05018 InsertPos = 0; 05019 05020 // Now we update the operands. 05021 for (unsigned i = 0; i != NumOps; ++i) 05022 if (N->OperandList[i] != Ops[i]) 05023 N->OperandList[i].set(Ops[i]); 05024 05025 // If this gets put into a CSE map, add it. 05026 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 05027 return N; 05028 } 05029 05030 /// DropOperands - Release the operands and set this node to have 05031 /// zero operands. 05032 void SDNode::DropOperands() { 05033 // Unlike the code in MorphNodeTo that does this, we don't need to 05034 // watch for dead nodes here. 05035 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 05036 SDUse &Use = *I++; 05037 Use.set(SDValue()); 05038 } 05039 } 05040 05041 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 05042 /// machine opcode. 05043 /// 05044 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05045 EVT VT) { 05046 SDVTList VTs = getVTList(VT); 05047 return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 05048 } 05049 05050 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05051 EVT VT, SDValue Op1) { 05052 SDVTList VTs = getVTList(VT); 05053 SDValue Ops[] = { Op1 }; 05054 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 05055 } 05056 05057 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05058 EVT VT, SDValue Op1, 05059 SDValue Op2) { 05060 SDVTList VTs = getVTList(VT); 05061 SDValue Ops[] = { Op1, Op2 }; 05062 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 05063 } 05064 05065 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05066 EVT VT, SDValue Op1, 05067 SDValue Op2, SDValue Op3) { 05068 SDVTList VTs = getVTList(VT); 05069 SDValue Ops[] = { Op1, Op2, Op3 }; 05070 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 05071 } 05072 05073 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05074 EVT VT, const SDValue *Ops, 05075 unsigned NumOps) { 05076 SDVTList VTs = getVTList(VT); 05077 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 05078 } 05079 05080 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05081 EVT VT1, EVT VT2, const SDValue *Ops, 05082 unsigned NumOps) { 05083 SDVTList VTs = getVTList(VT1, VT2); 05084 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 05085 } 05086 05087 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05088 EVT VT1, EVT VT2) { 05089 SDVTList VTs = getVTList(VT1, VT2); 05090 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 05091 } 05092 05093 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05094 EVT VT1, EVT VT2, EVT VT3, 05095 const SDValue *Ops, unsigned NumOps) { 05096 SDVTList VTs = getVTList(VT1, VT2, VT3); 05097 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 05098 } 05099 05100 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05101 EVT VT1, EVT VT2, EVT VT3, EVT VT4, 05102 const SDValue *Ops, unsigned NumOps) { 05103 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 05104 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 05105 } 05106 05107 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05108 EVT VT1, EVT VT2, 05109 SDValue Op1) { 05110 SDVTList VTs = getVTList(VT1, VT2); 05111 SDValue Ops[] = { Op1 }; 05112 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 05113 } 05114 05115 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05116 EVT VT1, EVT VT2, 05117 SDValue Op1, SDValue Op2) { 05118 SDVTList VTs = getVTList(VT1, VT2); 05119 SDValue Ops[] = { Op1, Op2 }; 05120 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 05121 } 05122 05123 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05124 EVT VT1, EVT VT2, 05125 SDValue Op1, SDValue Op2, 05126 SDValue Op3) { 05127 SDVTList VTs = getVTList(VT1, VT2); 05128 SDValue Ops[] = { Op1, Op2, Op3 }; 05129 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 05130 } 05131 05132 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05133 EVT VT1, EVT VT2, EVT VT3, 05134 SDValue Op1, SDValue Op2, 05135 SDValue Op3) { 05136 SDVTList VTs = getVTList(VT1, VT2, VT3); 05137 SDValue Ops[] = { Op1, Op2, Op3 }; 05138 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 05139 } 05140 05141 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 05142 SDVTList VTs, const SDValue *Ops, 05143 unsigned NumOps) { 05144 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 05145 // Reset the NodeID to -1. 05146 N->setNodeId(-1); 05147 return N; 05148 } 05149 05150 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away 05151 /// the line number information on the merged node since it is not possible to 05152 /// preserve the information that operation is associated with multiple lines. 05153 /// This will make the debugger working better at -O0, were there is a higher 05154 /// probability having other instructions associated with that line. 05155 /// 05156 /// For IROrder, we keep the smaller of the two 05157 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { 05158 DebugLoc NLoc = N->getDebugLoc(); 05159 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && 05160 (OLoc.getDebugLoc() != NLoc)) { 05161 N->setDebugLoc(DebugLoc()); 05162 } 05163 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 05164 N->setIROrder(Order); 05165 return N; 05166 } 05167 05168 /// MorphNodeTo - This *mutates* the specified node to have the specified 05169 /// return type, opcode, and operands. 05170 /// 05171 /// Note that MorphNodeTo returns the resultant node. If there is already a 05172 /// node of the specified opcode and operands, it returns that node instead of 05173 /// the current one. Note that the SDLoc need not be the same. 05174 /// 05175 /// Using MorphNodeTo is faster than creating a new node and swapping it in 05176 /// with ReplaceAllUsesWith both because it often avoids allocating a new 05177 /// node, and because it doesn't require CSE recalculation for any of 05178 /// the node's users. 05179 /// 05180 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 05181 SDVTList VTs, const SDValue *Ops, 05182 unsigned NumOps) { 05183 // If an identical node already exists, use it. 05184 void *IP = 0; 05185 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 05186 FoldingSetNodeID ID; 05187 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 05188 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 05189 return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N)); 05190 } 05191 05192 if (!RemoveNodeFromCSEMaps(N)) 05193 IP = 0; 05194 05195 // Start the morphing. 05196 N->NodeType = Opc; 05197 N->ValueList = VTs.VTs; 05198 N->NumValues = VTs.NumVTs; 05199 05200 // Clear the operands list, updating used nodes to remove this from their 05201 // use list. Keep track of any operands that become dead as a result. 05202 SmallPtrSet<SDNode*, 16> DeadNodeSet; 05203 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 05204 SDUse &Use = *I++; 05205 SDNode *Used = Use.getNode(); 05206 Use.set(SDValue()); 05207 if (Used->use_empty()) 05208 DeadNodeSet.insert(Used); 05209 } 05210 05211 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 05212 // Initialize the memory references information. 05213 MN->setMemRefs(0, 0); 05214 // If NumOps is larger than the # of operands we can have in a 05215 // MachineSDNode, reallocate the operand list. 05216 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 05217 if (MN->OperandsNeedDelete) 05218 delete[] MN->OperandList; 05219 if (NumOps > array_lengthof(MN->LocalOperands)) 05220 // We're creating a final node that will live unmorphed for the 05221 // remainder of the current SelectionDAG iteration, so we can allocate 05222 // the operands directly out of a pool with no recycling metadata. 05223 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 05224 Ops, NumOps); 05225 else 05226 MN->InitOperands(MN->LocalOperands, Ops, NumOps); 05227 MN->OperandsNeedDelete = false; 05228 } else 05229 MN->InitOperands(MN->OperandList, Ops, NumOps); 05230 } else { 05231 // If NumOps is larger than the # of operands we currently have, reallocate 05232 // the operand list. 05233 if (NumOps > N->NumOperands) { 05234 if (N->OperandsNeedDelete) 05235 delete[] N->OperandList; 05236 N->InitOperands(new SDUse[NumOps], Ops, NumOps); 05237 N->OperandsNeedDelete = true; 05238 } else 05239 N->InitOperands(N->OperandList, Ops, NumOps); 05240 } 05241 05242 // Delete any nodes that are still dead after adding the uses for the 05243 // new operands. 05244 if (!DeadNodeSet.empty()) { 05245 SmallVector<SDNode *, 16> DeadNodes; 05246 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 05247 E = DeadNodeSet.end(); I != E; ++I) 05248 if ((*I)->use_empty()) 05249 DeadNodes.push_back(*I); 05250 RemoveDeadNodes(DeadNodes); 05251 } 05252 05253 if (IP) 05254 CSEMap.InsertNode(N, IP); // Memoize the new node. 05255 return N; 05256 } 05257 05258 05259 /// getMachineNode - These are used for target selectors to create a new node 05260 /// with specified return type(s), MachineInstr opcode, and operands. 05261 /// 05262 /// Note that getMachineNode returns the resultant node. If there is already a 05263 /// node of the specified opcode and operands, it returns that node instead of 05264 /// the current one. 05265 MachineSDNode * 05266 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) { 05267 SDVTList VTs = getVTList(VT); 05268 return getMachineNode(Opcode, dl, VTs, None); 05269 } 05270 05271 MachineSDNode * 05272 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) { 05273 SDVTList VTs = getVTList(VT); 05274 SDValue Ops[] = { Op1 }; 05275 return getMachineNode(Opcode, dl, VTs, Ops); 05276 } 05277 05278 MachineSDNode * 05279 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 05280 SDValue Op1, SDValue Op2) { 05281 SDVTList VTs = getVTList(VT); 05282 SDValue Ops[] = { Op1, Op2 }; 05283 return getMachineNode(Opcode, dl, VTs, Ops); 05284 } 05285 05286 MachineSDNode * 05287 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 05288 SDValue Op1, SDValue Op2, SDValue Op3) { 05289 SDVTList VTs = getVTList(VT); 05290 SDValue Ops[] = { Op1, Op2, Op3 }; 05291 return getMachineNode(Opcode, dl, VTs, Ops); 05292 } 05293 05294 MachineSDNode * 05295 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 05296 ArrayRef<SDValue> Ops) { 05297 SDVTList VTs = getVTList(VT); 05298 return getMachineNode(Opcode, dl, VTs, Ops); 05299 } 05300 05301 MachineSDNode * 05302 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) { 05303 SDVTList VTs = getVTList(VT1, VT2); 05304 return getMachineNode(Opcode, dl, VTs, None); 05305 } 05306 05307 MachineSDNode * 05308 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05309 EVT VT1, EVT VT2, SDValue Op1) { 05310 SDVTList VTs = getVTList(VT1, VT2); 05311 SDValue Ops[] = { Op1 }; 05312 return getMachineNode(Opcode, dl, VTs, Ops); 05313 } 05314 05315 MachineSDNode * 05316 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05317 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 05318 SDVTList VTs = getVTList(VT1, VT2); 05319 SDValue Ops[] = { Op1, Op2 }; 05320 return getMachineNode(Opcode, dl, VTs, Ops); 05321 } 05322 05323 MachineSDNode * 05324 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05325 EVT VT1, EVT VT2, SDValue Op1, 05326 SDValue Op2, SDValue Op3) { 05327 SDVTList VTs = getVTList(VT1, VT2); 05328 SDValue Ops[] = { Op1, Op2, Op3 }; 05329 return getMachineNode(Opcode, dl, VTs, Ops); 05330 } 05331 05332 MachineSDNode * 05333 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05334 EVT VT1, EVT VT2, 05335 ArrayRef<SDValue> Ops) { 05336 SDVTList VTs = getVTList(VT1, VT2); 05337 return getMachineNode(Opcode, dl, VTs, Ops); 05338 } 05339 05340 MachineSDNode * 05341 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05342 EVT VT1, EVT VT2, EVT VT3, 05343 SDValue Op1, SDValue Op2) { 05344 SDVTList VTs = getVTList(VT1, VT2, VT3); 05345 SDValue Ops[] = { Op1, Op2 }; 05346 return getMachineNode(Opcode, dl, VTs, Ops); 05347 } 05348 05349 MachineSDNode * 05350 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05351 EVT VT1, EVT VT2, EVT VT3, 05352 SDValue Op1, SDValue Op2, SDValue Op3) { 05353 SDVTList VTs = getVTList(VT1, VT2, VT3); 05354 SDValue Ops[] = { Op1, Op2, Op3 }; 05355 return getMachineNode(Opcode, dl, VTs, Ops); 05356 } 05357 05358 MachineSDNode * 05359 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05360 EVT VT1, EVT VT2, EVT VT3, 05361 ArrayRef<SDValue> Ops) { 05362 SDVTList VTs = getVTList(VT1, VT2, VT3); 05363 return getMachineNode(Opcode, dl, VTs, Ops); 05364 } 05365 05366 MachineSDNode * 05367 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, 05368 EVT VT2, EVT VT3, EVT VT4, 05369 ArrayRef<SDValue> Ops) { 05370 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 05371 return getMachineNode(Opcode, dl, VTs, Ops); 05372 } 05373 05374 MachineSDNode * 05375 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 05376 ArrayRef<EVT> ResultTys, 05377 ArrayRef<SDValue> Ops) { 05378 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 05379 return getMachineNode(Opcode, dl, VTs, Ops); 05380 } 05381 05382 MachineSDNode * 05383 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, 05384 ArrayRef<SDValue> OpsArray) { 05385 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 05386 MachineSDNode *N; 05387 void *IP = 0; 05388 const SDValue *Ops = OpsArray.data(); 05389 unsigned NumOps = OpsArray.size(); 05390 05391 if (DoCSE) { 05392 FoldingSetNodeID ID; 05393 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 05394 IP = 0; 05395 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 05396 return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); 05397 } 05398 } 05399 05400 // Allocate a new MachineSDNode. 05401 N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 05402 05403 // Initialize the operands list. 05404 if (NumOps > array_lengthof(N->LocalOperands)) 05405 // We're creating a final node that will live unmorphed for the 05406 // remainder of the current SelectionDAG iteration, so we can allocate 05407 // the operands directly out of a pool with no recycling metadata. 05408 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 05409 Ops, NumOps); 05410 else 05411 N->InitOperands(N->LocalOperands, Ops, NumOps); 05412 N->OperandsNeedDelete = false; 05413 05414 if (DoCSE) 05415 CSEMap.InsertNode(N, IP); 05416 05417 AllNodes.push_back(N); 05418 #ifndef NDEBUG 05419 VerifyMachineNode(N); 05420 #endif 05421 return N; 05422 } 05423 05424 /// getTargetExtractSubreg - A convenience function for creating 05425 /// TargetOpcode::EXTRACT_SUBREG nodes. 05426 SDValue 05427 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, 05428 SDValue Operand) { 05429 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 05430 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 05431 VT, Operand, SRIdxVal); 05432 return SDValue(Subreg, 0); 05433 } 05434 05435 /// getTargetInsertSubreg - A convenience function for creating 05436 /// TargetOpcode::INSERT_SUBREG nodes. 05437 SDValue 05438 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, 05439 SDValue Operand, SDValue Subreg) { 05440 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 05441 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 05442 VT, Operand, Subreg, SRIdxVal); 05443 return SDValue(Result, 0); 05444 } 05445 05446 /// getNodeIfExists - Get the specified node if it's already available, or 05447 /// else return NULL. 05448 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 05449 const SDValue *Ops, unsigned NumOps) { 05450 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 05451 FoldingSetNodeID ID; 05452 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 05453 void *IP = 0; 05454 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 05455 return E; 05456 } 05457 return NULL; 05458 } 05459 05460 /// getDbgValue - Creates a SDDbgValue node. 05461 /// 05462 SDDbgValue * 05463 SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 05464 DebugLoc DL, unsigned O) { 05465 return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 05466 } 05467 05468 SDDbgValue * 05469 SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 05470 DebugLoc DL, unsigned O) { 05471 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 05472 } 05473 05474 SDDbgValue * 05475 SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 05476 DebugLoc DL, unsigned O) { 05477 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 05478 } 05479 05480 namespace { 05481 05482 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 05483 /// pointed to by a use iterator is deleted, increment the use iterator 05484 /// so that it doesn't dangle. 05485 /// 05486 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 05487 SDNode::use_iterator &UI; 05488 SDNode::use_iterator &UE; 05489 05490 virtual void NodeDeleted(SDNode *N, SDNode *E) { 05491 // Increment the iterator as needed. 05492 while (UI != UE && N == *UI) 05493 ++UI; 05494 } 05495 05496 public: 05497 RAUWUpdateListener(SelectionDAG &d, 05498 SDNode::use_iterator &ui, 05499 SDNode::use_iterator &ue) 05500 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 05501 }; 05502 05503 } 05504 05505 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 05506 /// This can cause recursive merging of nodes in the DAG. 05507 /// 05508 /// This version assumes From has a single result value. 05509 /// 05510 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 05511 SDNode *From = FromN.getNode(); 05512 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 05513 "Cannot replace with this method!"); 05514 assert(From != To.getNode() && "Cannot replace uses of with self"); 05515 05516 // Iterate over all the existing uses of From. New uses will be added 05517 // to the beginning of the use list, which we avoid visiting. 05518 // This specifically avoids visiting uses of From that arise while the 05519 // replacement is happening, because any such uses would be the result 05520 // of CSE: If an existing node looks like From after one of its operands 05521 // is replaced by To, we don't want to replace of all its users with To 05522 // too. See PR3018 for more info. 05523 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 05524 RAUWUpdateListener Listener(*this, UI, UE); 05525 while (UI != UE) { 05526 SDNode *User = *UI; 05527 05528 // This node is about to morph, remove its old self from the CSE maps. 05529 RemoveNodeFromCSEMaps(User); 05530 05531 // A user can appear in a use list multiple times, and when this 05532 // happens the uses are usually next to each other in the list. 05533 // To help reduce the number of CSE recomputations, process all 05534 // the uses of this user that we can find this way. 05535 do { 05536 SDUse &Use = UI.getUse(); 05537 ++UI; 05538 Use.set(To); 05539 } while (UI != UE && *UI == User); 05540 05541 // Now that we have modified User, add it back to the CSE maps. If it 05542 // already exists there, recursively merge the results together. 05543 AddModifiedNodeToCSEMaps(User); 05544 } 05545 05546 // If we just RAUW'd the root, take note. 05547 if (FromN == getRoot()) 05548 setRoot(To); 05549 } 05550 05551 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 05552 /// This can cause recursive merging of nodes in the DAG. 05553 /// 05554 /// This version assumes that for each value of From, there is a 05555 /// corresponding value in To in the same position with the same type. 05556 /// 05557 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 05558 #ifndef NDEBUG 05559 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 05560 assert((!From->hasAnyUseOfValue(i) || 05561 From->getValueType(i) == To->getValueType(i)) && 05562 "Cannot use this version of ReplaceAllUsesWith!"); 05563 #endif 05564 05565 // Handle the trivial case. 05566 if (From == To) 05567 return; 05568 05569 // Iterate over just the existing users of From. See the comments in 05570 // the ReplaceAllUsesWith above. 05571 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 05572 RAUWUpdateListener Listener(*this, UI, UE); 05573 while (UI != UE) { 05574 SDNode *User = *UI; 05575 05576 // This node is about to morph, remove its old self from the CSE maps. 05577 RemoveNodeFromCSEMaps(User); 05578 05579 // A user can appear in a use list multiple times, and when this 05580 // happens the uses are usually next to each other in the list. 05581 // To help reduce the number of CSE recomputations, process all 05582 // the uses of this user that we can find this way. 05583 do { 05584 SDUse &Use = UI.getUse(); 05585 ++UI; 05586 Use.setNode(To); 05587 } while (UI != UE && *UI == User); 05588 05589 // Now that we have modified User, add it back to the CSE maps. If it 05590 // already exists there, recursively merge the results together. 05591 AddModifiedNodeToCSEMaps(User); 05592 } 05593 05594 // If we just RAUW'd the root, take note. 05595 if (From == getRoot().getNode()) 05596 setRoot(SDValue(To, getRoot().getResNo())); 05597 } 05598 05599 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 05600 /// This can cause recursive merging of nodes in the DAG. 05601 /// 05602 /// This version can replace From with any result values. To must match the 05603 /// number and types of values returned by From. 05604 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 05605 if (From->getNumValues() == 1) // Handle the simple case efficiently. 05606 return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 05607 05608 // Iterate over just the existing users of From. See the comments in 05609 // the ReplaceAllUsesWith above. 05610 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 05611 RAUWUpdateListener Listener(*this, UI, UE); 05612 while (UI != UE) { 05613 SDNode *User = *UI; 05614 05615 // This node is about to morph, remove its old self from the CSE maps. 05616 RemoveNodeFromCSEMaps(User); 05617 05618 // A user can appear in a use list multiple times, and when this 05619 // happens the uses are usually next to each other in the list. 05620 // To help reduce the number of CSE recomputations, process all 05621 // the uses of this user that we can find this way. 05622 do { 05623 SDUse &Use = UI.getUse(); 05624 const SDValue &ToOp = To[Use.getResNo()]; 05625 ++UI; 05626 Use.set(ToOp); 05627 } while (UI != UE && *UI == User); 05628 05629 // Now that we have modified User, add it back to the CSE maps. If it 05630 // already exists there, recursively merge the results together. 05631 AddModifiedNodeToCSEMaps(User); 05632 } 05633 05634 // If we just RAUW'd the root, take note. 05635 if (From == getRoot().getNode()) 05636 setRoot(SDValue(To[getRoot().getResNo()])); 05637 } 05638 05639 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 05640 /// uses of other values produced by From.getNode() alone. The Deleted 05641 /// vector is handled the same way as for ReplaceAllUsesWith. 05642 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 05643 // Handle the really simple, really trivial case efficiently. 05644 if (From == To) return; 05645 05646 // Handle the simple, trivial, case efficiently. 05647 if (From.getNode()->getNumValues() == 1) { 05648 ReplaceAllUsesWith(From, To); 05649 return; 05650 } 05651 05652 // Iterate over just the existing users of From. See the comments in 05653 // the ReplaceAllUsesWith above. 05654 SDNode::use_iterator UI = From.getNode()->use_begin(), 05655 UE = From.getNode()->use_end(); 05656 RAUWUpdateListener Listener(*this, UI, UE); 05657 while (UI != UE) { 05658 SDNode *User = *UI; 05659 bool UserRemovedFromCSEMaps = false; 05660 05661 // A user can appear in a use list multiple times, and when this 05662 // happens the uses are usually next to each other in the list. 05663 // To help reduce the number of CSE recomputations, process all 05664 // the uses of this user that we can find this way. 05665 do { 05666 SDUse &Use = UI.getUse(); 05667 05668 // Skip uses of different values from the same node. 05669 if (Use.getResNo() != From.getResNo()) { 05670 ++UI; 05671 continue; 05672 } 05673 05674 // If this node hasn't been modified yet, it's still in the CSE maps, 05675 // so remove its old self from the CSE maps. 05676 if (!UserRemovedFromCSEMaps) { 05677 RemoveNodeFromCSEMaps(User); 05678 UserRemovedFromCSEMaps = true; 05679 } 05680 05681 ++UI; 05682 Use.set(To); 05683 } while (UI != UE && *UI == User); 05684 05685 // We are iterating over all uses of the From node, so if a use 05686 // doesn't use the specific value, no changes are made. 05687 if (!UserRemovedFromCSEMaps) 05688 continue; 05689 05690 // Now that we have modified User, add it back to the CSE maps. If it 05691 // already exists there, recursively merge the results together. 05692 AddModifiedNodeToCSEMaps(User); 05693 } 05694 05695 // If we just RAUW'd the root, take note. 05696 if (From == getRoot()) 05697 setRoot(To); 05698 } 05699 05700 namespace { 05701 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 05702 /// to record information about a use. 05703 struct UseMemo { 05704 SDNode *User; 05705 unsigned Index; 05706 SDUse *Use; 05707 }; 05708 05709 /// operator< - Sort Memos by User. 05710 bool operator<(const UseMemo &L, const UseMemo &R) { 05711 return (intptr_t)L.User < (intptr_t)R.User; 05712 } 05713 } 05714 05715 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 05716 /// uses of other values produced by From.getNode() alone. The same value 05717 /// may appear in both the From and To list. The Deleted vector is 05718 /// handled the same way as for ReplaceAllUsesWith. 05719 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 05720 const SDValue *To, 05721 unsigned Num){ 05722 // Handle the simple, trivial case efficiently. 05723 if (Num == 1) 05724 return ReplaceAllUsesOfValueWith(*From, *To); 05725 05726 // Read up all the uses and make records of them. This helps 05727 // processing new uses that are introduced during the 05728 // replacement process. 05729 SmallVector<UseMemo, 4> Uses; 05730 for (unsigned i = 0; i != Num; ++i) { 05731 unsigned FromResNo = From[i].getResNo(); 05732 SDNode *FromNode = From[i].getNode(); 05733 for (SDNode::use_iterator UI = FromNode->use_begin(), 05734 E = FromNode->use_end(); UI != E; ++UI) { 05735 SDUse &Use = UI.getUse(); 05736 if (Use.getResNo() == FromResNo) { 05737 UseMemo Memo = { *UI, i, &Use }; 05738 Uses.push_back(Memo); 05739 } 05740 } 05741 } 05742 05743 // Sort the uses, so that all the uses from a given User are together. 05744 std::sort(Uses.begin(), Uses.end()); 05745 05746 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 05747 UseIndex != UseIndexEnd; ) { 05748 // We know that this user uses some value of From. If it is the right 05749 // value, update it. 05750 SDNode *User = Uses[UseIndex].User; 05751 05752 // This node is about to morph, remove its old self from the CSE maps. 05753 RemoveNodeFromCSEMaps(User); 05754 05755 // The Uses array is sorted, so all the uses for a given User 05756 // are next to each other in the list. 05757 // To help reduce the number of CSE recomputations, process all 05758 // the uses of this user that we can find this way. 05759 do { 05760 unsigned i = Uses[UseIndex].Index; 05761 SDUse &Use = *Uses[UseIndex].Use; 05762 ++UseIndex; 05763 05764 Use.set(To[i]); 05765 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 05766 05767 // Now that we have modified User, add it back to the CSE maps. If it 05768 // already exists there, recursively merge the results together. 05769 AddModifiedNodeToCSEMaps(User); 05770 } 05771 } 05772 05773 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 05774 /// based on their topological order. It returns the maximum id and a vector 05775 /// of the SDNodes* in assigned order by reference. 05776 unsigned SelectionDAG::AssignTopologicalOrder() { 05777 05778 unsigned DAGSize = 0; 05779 05780 // SortedPos tracks the progress of the algorithm. Nodes before it are 05781 // sorted, nodes after it are unsorted. When the algorithm completes 05782 // it is at the end of the list. 05783 allnodes_iterator SortedPos = allnodes_begin(); 05784 05785 // Visit all the nodes. Move nodes with no operands to the front of 05786 // the list immediately. Annotate nodes that do have operands with their 05787 // operand count. Before we do this, the Node Id fields of the nodes 05788 // may contain arbitrary values. After, the Node Id fields for nodes 05789 // before SortedPos will contain the topological sort index, and the 05790 // Node Id fields for nodes At SortedPos and after will contain the 05791 // count of outstanding operands. 05792 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 05793 SDNode *N = I++; 05794 checkForCycles(N); 05795 unsigned Degree = N->getNumOperands(); 05796 if (Degree == 0) { 05797 // A node with no uses, add it to the result array immediately. 05798 N->setNodeId(DAGSize++); 05799 allnodes_iterator Q = N; 05800 if (Q != SortedPos) 05801 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 05802 assert(SortedPos != AllNodes.end() && "Overran node list"); 05803 ++SortedPos; 05804 } else { 05805 // Temporarily use the Node Id as scratch space for the degree count. 05806 N->setNodeId(Degree); 05807 } 05808 } 05809 05810 // Visit all the nodes. As we iterate, move nodes into sorted order, 05811 // such that by the time the end is reached all nodes will be sorted. 05812 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 05813 SDNode *N = I; 05814 checkForCycles(N); 05815 // N is in sorted position, so all its uses have one less operand 05816 // that needs to be sorted. 05817 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 05818 UI != UE; ++UI) { 05819 SDNode *P = *UI; 05820 unsigned Degree = P->getNodeId(); 05821 assert(Degree != 0 && "Invalid node degree"); 05822 --Degree; 05823 if (Degree == 0) { 05824 // All of P's operands are sorted, so P may sorted now. 05825 P->setNodeId(DAGSize++); 05826 if (P != SortedPos) 05827 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 05828 assert(SortedPos != AllNodes.end() && "Overran node list"); 05829 ++SortedPos; 05830 } else { 05831 // Update P's outstanding operand count. 05832 P->setNodeId(Degree); 05833 } 05834 } 05835 if (I == SortedPos) { 05836 #ifndef NDEBUG 05837 SDNode *S = ++I; 05838 dbgs() << "Overran sorted position:\n"; 05839 S->dumprFull(); 05840 #endif 05841 llvm_unreachable(0); 05842 } 05843 } 05844 05845 assert(SortedPos == AllNodes.end() && 05846 "Topological sort incomplete!"); 05847 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 05848 "First node in topological sort is not the entry token!"); 05849 assert(AllNodes.front().getNodeId() == 0 && 05850 "First node in topological sort has non-zero id!"); 05851 assert(AllNodes.front().getNumOperands() == 0 && 05852 "First node in topological sort has operands!"); 05853 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 05854 "Last node in topologic sort has unexpected id!"); 05855 assert(AllNodes.back().use_empty() && 05856 "Last node in topologic sort has users!"); 05857 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 05858 return DAGSize; 05859 } 05860 05861 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 05862 /// value is produced by SD. 05863 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 05864 DbgInfo->add(DB, SD, isParameter); 05865 if (SD) 05866 SD->setHasDebugValue(true); 05867 } 05868 05869 /// TransferDbgValues - Transfer SDDbgValues. 05870 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 05871 if (From == To || !From.getNode()->getHasDebugValue()) 05872 return; 05873 SDNode *FromNode = From.getNode(); 05874 SDNode *ToNode = To.getNode(); 05875 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 05876 SmallVector<SDDbgValue *, 2> ClonedDVs; 05877 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 05878 I != E; ++I) { 05879 SDDbgValue *Dbg = *I; 05880 if (Dbg->getKind() == SDDbgValue::SDNODE) { 05881 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 05882 Dbg->getOffset(), Dbg->getDebugLoc(), 05883 Dbg->getOrder()); 05884 ClonedDVs.push_back(Clone); 05885 } 05886 } 05887 for (SmallVector<SDDbgValue *, 2>::iterator I = ClonedDVs.begin(), 05888 E = ClonedDVs.end(); I != E; ++I) 05889 AddDbgValue(*I, ToNode, false); 05890 } 05891 05892 //===----------------------------------------------------------------------===// 05893 // SDNode Class 05894 //===----------------------------------------------------------------------===// 05895 05896 HandleSDNode::~HandleSDNode() { 05897 DropOperands(); 05898 } 05899 05900 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 05901 DebugLoc DL, const GlobalValue *GA, 05902 EVT VT, int64_t o, unsigned char TF) 05903 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 05904 TheGlobal = GA; 05905 } 05906 05907 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 05908 EVT memvt, MachineMemOperand *mmo) 05909 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 05910 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 05911 MMO->isNonTemporal(), MMO->isInvariant()); 05912 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 05913 assert(isNonTemporal() == MMO->isNonTemporal() && 05914 "Non-temporal encoding error!"); 05915 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 05916 } 05917 05918 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 05919 const SDValue *Ops, unsigned NumOps, EVT memvt, 05920 MachineMemOperand *mmo) 05921 : SDNode(Opc, Order, dl, VTs, Ops, NumOps), 05922 MemoryVT(memvt), MMO(mmo) { 05923 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 05924 MMO->isNonTemporal(), MMO->isInvariant()); 05925 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 05926 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 05927 } 05928 05929 /// Profile - Gather unique data for the node. 05930 /// 05931 void SDNode::Profile(FoldingSetNodeID &ID) const { 05932 AddNodeIDNode(ID, this); 05933 } 05934 05935 namespace { 05936 struct EVTArray { 05937 std::vector<EVT> VTs; 05938 05939 EVTArray() { 05940 VTs.reserve(MVT::LAST_VALUETYPE); 05941 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 05942 VTs.push_back(MVT((MVT::SimpleValueType)i)); 05943 } 05944 }; 05945 } 05946 05947 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 05948 static ManagedStatic<EVTArray> SimpleVTArray; 05949 static ManagedStatic<sys::SmartMutex<true> > VTMutex; 05950 05951 /// getValueTypeList - Return a pointer to the specified value type. 05952 /// 05953 const EVT *SDNode::getValueTypeList(EVT VT) { 05954 if (VT.isExtended()) { 05955 sys::SmartScopedLock<true> Lock(*VTMutex); 05956 return &(*EVTs->insert(VT).first); 05957 } else { 05958 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 05959 "Value type out of range!"); 05960 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 05961 } 05962 } 05963 05964 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 05965 /// indicated value. This method ignores uses of other values defined by this 05966 /// operation. 05967 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 05968 assert(Value < getNumValues() && "Bad value!"); 05969 05970 // TODO: Only iterate over uses of a given value of the node 05971 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 05972 if (UI.getUse().getResNo() == Value) { 05973 if (NUses == 0) 05974 return false; 05975 --NUses; 05976 } 05977 } 05978 05979 // Found exactly the right number of uses? 05980 return NUses == 0; 05981 } 05982 05983 05984 /// hasAnyUseOfValue - Return true if there are any use of the indicated 05985 /// value. This method ignores uses of other values defined by this operation. 05986 bool SDNode::hasAnyUseOfValue(unsigned Value) const { 05987 assert(Value < getNumValues() && "Bad value!"); 05988 05989 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 05990 if (UI.getUse().getResNo() == Value) 05991 return true; 05992 05993 return false; 05994 } 05995 05996 05997 /// isOnlyUserOf - Return true if this node is the only use of N. 05998 /// 05999 bool SDNode::isOnlyUserOf(SDNode *N) const { 06000 bool Seen = false; 06001 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 06002 SDNode *User = *I; 06003 if (User == this) 06004 Seen = true; 06005 else 06006 return false; 06007 } 06008 06009 return Seen; 06010 } 06011 06012 /// isOperand - Return true if this node is an operand of N. 06013 /// 06014 bool SDValue::isOperandOf(SDNode *N) const { 06015 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 06016 if (*this == N->getOperand(i)) 06017 return true; 06018 return false; 06019 } 06020 06021 bool SDNode::isOperandOf(SDNode *N) const { 06022 for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 06023 if (this == N->OperandList[i].getNode()) 06024 return true; 06025 return false; 06026 } 06027 06028 /// reachesChainWithoutSideEffects - Return true if this operand (which must 06029 /// be a chain) reaches the specified operand without crossing any 06030 /// side-effecting instructions on any chain path. In practice, this looks 06031 /// through token factors and non-volatile loads. In order to remain efficient, 06032 /// this only looks a couple of nodes in, it does not do an exhaustive search. 06033 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 06034 unsigned Depth) const { 06035 if (*this == Dest) return true; 06036 06037 // Don't search too deeply, we just want to be able to see through 06038 // TokenFactor's etc. 06039 if (Depth == 0) return false; 06040 06041 // If this is a token factor, all inputs to the TF happen in parallel. If any 06042 // of the operands of the TF does not reach dest, then we cannot do the xform. 06043 if (getOpcode() == ISD::TokenFactor) { 06044 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 06045 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 06046 return false; 06047 return true; 06048 } 06049 06050 // Loads don't have side effects, look through them. 06051 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 06052 if (!Ld->isVolatile()) 06053 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 06054 } 06055 return false; 06056 } 06057 06058 /// hasPredecessor - Return true if N is a predecessor of this node. 06059 /// N is either an operand of this node, or can be reached by recursively 06060 /// traversing up the operands. 06061 /// NOTE: This is an expensive method. Use it carefully. 06062 bool SDNode::hasPredecessor(const SDNode *N) const { 06063 SmallPtrSet<const SDNode *, 32> Visited; 06064 SmallVector<const SDNode *, 16> Worklist; 06065 return hasPredecessorHelper(N, Visited, Worklist); 06066 } 06067 06068 bool SDNode::hasPredecessorHelper(const SDNode *N, 06069 SmallPtrSet<const SDNode *, 32> &Visited, 06070 SmallVector<const SDNode *, 16> &Worklist) const { 06071 if (Visited.empty()) { 06072 Worklist.push_back(this); 06073 } else { 06074 // Take a look in the visited set. If we've already encountered this node 06075 // we needn't search further. 06076 if (Visited.count(N)) 06077 return true; 06078 } 06079 06080 // Haven't visited N yet. Continue the search. 06081 while (!Worklist.empty()) { 06082 const SDNode *M = Worklist.pop_back_val(); 06083 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 06084 SDNode *Op = M->getOperand(i).getNode(); 06085 if (Visited.insert(Op)) 06086 Worklist.push_back(Op); 06087 if (Op == N) 06088 return true; 06089 } 06090 } 06091 06092 return false; 06093 } 06094 06095 uint64_t SDNode::getConstantOperandVal(unsigned Num) const { 06096 assert(Num < NumOperands && "Invalid child # of SDNode!"); 06097 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 06098 } 06099 06100 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 06101 assert(N->getNumValues() == 1 && 06102 "Can't unroll a vector with multiple results!"); 06103 06104 EVT VT = N->getValueType(0); 06105 unsigned NE = VT.getVectorNumElements(); 06106 EVT EltVT = VT.getVectorElementType(); 06107 SDLoc dl(N); 06108 06109 SmallVector<SDValue, 8> Scalars; 06110 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 06111 06112 // If ResNE is 0, fully unroll the vector op. 06113 if (ResNE == 0) 06114 ResNE = NE; 06115 else if (NE > ResNE) 06116 NE = ResNE; 06117 06118 unsigned i; 06119 for (i= 0; i != NE; ++i) { 06120 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 06121 SDValue Operand = N->getOperand(j); 06122 EVT OperandVT = Operand.getValueType(); 06123 if (OperandVT.isVector()) { 06124 // A vector operand; extract a single element. 06125 EVT OperandEltVT = OperandVT.getVectorElementType(); 06126 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 06127 OperandEltVT, 06128 Operand, 06129 getConstant(i, TLI.getPointerTy())); 06130 } else { 06131 // A scalar operand; just use it as is. 06132 Operands[j] = Operand; 06133 } 06134 } 06135 06136 switch (N->getOpcode()) { 06137 default: 06138 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 06139 &Operands[0], Operands.size())); 06140 break; 06141 case ISD::VSELECT: 06142 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 06143 &Operands[0], Operands.size())); 06144 break; 06145 case ISD::SHL: 06146 case ISD::SRA: 06147 case ISD::SRL: 06148 case ISD::ROTL: 06149 case ISD::ROTR: 06150 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 06151 getShiftAmountOperand(Operands[0].getValueType(), 06152 Operands[1]))); 06153 break; 06154 case ISD::SIGN_EXTEND_INREG: 06155 case ISD::FP_ROUND_INREG: { 06156 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 06157 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 06158 Operands[0], 06159 getValueType(ExtVT))); 06160 } 06161 } 06162 } 06163 06164 for (; i < ResNE; ++i) 06165 Scalars.push_back(getUNDEF(EltVT)); 06166 06167 return getNode(ISD::BUILD_VECTOR, dl, 06168 EVT::getVectorVT(*getContext(), EltVT, ResNE), 06169 &Scalars[0], Scalars.size()); 06170 } 06171 06172 06173 /// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 06174 /// location that is 'Dist' units away from the location that the 'Base' load 06175 /// is loading from. 06176 bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 06177 unsigned Bytes, int Dist) const { 06178 if (LD->getChain() != Base->getChain()) 06179 return false; 06180 EVT VT = LD->getValueType(0); 06181 if (VT.getSizeInBits() / 8 != Bytes) 06182 return false; 06183 06184 SDValue Loc = LD->getOperand(1); 06185 SDValue BaseLoc = Base->getOperand(1); 06186 if (Loc.getOpcode() == ISD::FrameIndex) { 06187 if (BaseLoc.getOpcode() != ISD::FrameIndex) 06188 return false; 06189 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 06190 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 06191 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 06192 int FS = MFI->getObjectSize(FI); 06193 int BFS = MFI->getObjectSize(BFI); 06194 if (FS != BFS || FS != (int)Bytes) return false; 06195 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 06196 } 06197 06198 // Handle X+C 06199 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 06200 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 06201 return true; 06202 06203 const GlobalValue *GV1 = NULL; 06204 const GlobalValue *GV2 = NULL; 06205 int64_t Offset1 = 0; 06206 int64_t Offset2 = 0; 06207 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 06208 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 06209 if (isGA1 && isGA2 && GV1 == GV2) 06210 return Offset1 == (Offset2 + Dist*Bytes); 06211 return false; 06212 } 06213 06214 06215 /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 06216 /// it cannot be inferred. 06217 unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 06218 // If this is a GlobalAddress + cst, return the alignment. 06219 const GlobalValue *GV; 06220 int64_t GVOffset = 0; 06221 if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 06222 unsigned PtrWidth = TLI.getPointerTy().getSizeInBits(); 06223 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 06224 llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne, 06225 TLI.getDataLayout()); 06226 unsigned AlignBits = KnownZero.countTrailingOnes(); 06227 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 06228 if (Align) 06229 return MinAlign(Align, GVOffset); 06230 } 06231 06232 // If this is a direct reference to a stack slot, use information about the 06233 // stack slot's alignment. 06234 int FrameIdx = 1 << 31; 06235 int64_t FrameOffset = 0; 06236 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 06237 FrameIdx = FI->getIndex(); 06238 } else if (isBaseWithConstantOffset(Ptr) && 06239 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 06240 // Handle FI+Cst 06241 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 06242 FrameOffset = Ptr.getConstantOperandVal(1); 06243 } 06244 06245 if (FrameIdx != (1 << 31)) { 06246 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 06247 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 06248 FrameOffset); 06249 return FIInfoAlign; 06250 } 06251 06252 return 0; 06253 } 06254 06255 // getAddressSpace - Return the address space this GlobalAddress belongs to. 06256 unsigned GlobalAddressSDNode::getAddressSpace() const { 06257 return getGlobal()->getType()->getAddressSpace(); 06258 } 06259 06260 06261 Type *ConstantPoolSDNode::getType() const { 06262 if (isMachineConstantPoolEntry()) 06263 return Val.MachineCPVal->getType(); 06264 return Val.ConstVal->getType(); 06265 } 06266 06267 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 06268 APInt &SplatUndef, 06269 unsigned &SplatBitSize, 06270 bool &HasAnyUndefs, 06271 unsigned MinSplatBits, 06272 bool isBigEndian) { 06273 EVT VT = getValueType(0); 06274 assert(VT.isVector() && "Expected a vector type"); 06275 unsigned sz = VT.getSizeInBits(); 06276 if (MinSplatBits > sz) 06277 return false; 06278 06279 SplatValue = APInt(sz, 0); 06280 SplatUndef = APInt(sz, 0); 06281 06282 // Get the bits. Bits with undefined values (when the corresponding element 06283 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 06284 // in SplatValue. If any of the values are not constant, give up and return 06285 // false. 06286 unsigned int nOps = getNumOperands(); 06287 assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 06288 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 06289 06290 for (unsigned j = 0; j < nOps; ++j) { 06291 unsigned i = isBigEndian ? nOps-1-j : j; 06292 SDValue OpVal = getOperand(i); 06293 unsigned BitPos = j * EltBitSize; 06294 06295 if (OpVal.getOpcode() == ISD::UNDEF) 06296 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 06297 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 06298 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 06299 zextOrTrunc(sz) << BitPos; 06300 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 06301 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 06302 else 06303 return false; 06304 } 06305 06306 // The build_vector is all constants or undefs. Find the smallest element 06307 // size that splats the vector. 06308 06309 HasAnyUndefs = (SplatUndef != 0); 06310 while (sz > 8) { 06311 06312 unsigned HalfSize = sz / 2; 06313 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 06314 APInt LowValue = SplatValue.trunc(HalfSize); 06315 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 06316 APInt LowUndef = SplatUndef.trunc(HalfSize); 06317 06318 // If the two halves do not match (ignoring undef bits), stop here. 06319 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 06320 MinSplatBits > HalfSize) 06321 break; 06322 06323 SplatValue = HighValue | LowValue; 06324 SplatUndef = HighUndef & LowUndef; 06325 06326 sz = HalfSize; 06327 } 06328 06329 SplatBitSize = sz; 06330 return true; 06331 } 06332 06333 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 06334 // Find the first non-undef value in the shuffle mask. 06335 unsigned i, e; 06336 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) <