| File: | llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp |
| Warning: | line 3490, column 7 Called C++ object pointer is uninitialized |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- InstCombineCompares.cpp --------------------------------------------===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // This file implements the visitICmp and visitFCmp functions. | ||||||||
| 10 | // | ||||||||
| 11 | //===----------------------------------------------------------------------===// | ||||||||
| 12 | |||||||||
| 13 | #include "InstCombineInternal.h" | ||||||||
| 14 | #include "llvm/ADT/APSInt.h" | ||||||||
| 15 | #include "llvm/ADT/SetVector.h" | ||||||||
| 16 | #include "llvm/ADT/Statistic.h" | ||||||||
| 17 | #include "llvm/Analysis/ConstantFolding.h" | ||||||||
| 18 | #include "llvm/Analysis/InstructionSimplify.h" | ||||||||
| 19 | #include "llvm/Analysis/TargetLibraryInfo.h" | ||||||||
| 20 | #include "llvm/IR/ConstantRange.h" | ||||||||
| 21 | #include "llvm/IR/DataLayout.h" | ||||||||
| 22 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||||||
| 23 | #include "llvm/IR/IntrinsicInst.h" | ||||||||
| 24 | #include "llvm/IR/PatternMatch.h" | ||||||||
| 25 | #include "llvm/Support/Debug.h" | ||||||||
| 26 | #include "llvm/Support/KnownBits.h" | ||||||||
| 27 | |||||||||
| 28 | using namespace llvm; | ||||||||
| 29 | using namespace PatternMatch; | ||||||||
| 30 | |||||||||
| 31 | #define DEBUG_TYPE"instcombine" "instcombine" | ||||||||
| 32 | |||||||||
| 33 | // How many times is a select replaced by one of its operands? | ||||||||
| 34 | STATISTIC(NumSel, "Number of select opts")static llvm::Statistic NumSel = {"instcombine", "NumSel", "Number of select opts" }; | ||||||||
| 35 | |||||||||
| 36 | |||||||||
| 37 | /// Compute Result = In1+In2, returning true if the result overflowed for this | ||||||||
| 38 | /// type. | ||||||||
| 39 | static bool addWithOverflow(APInt &Result, const APInt &In1, | ||||||||
| 40 | const APInt &In2, bool IsSigned = false) { | ||||||||
| 41 | bool Overflow; | ||||||||
| 42 | if (IsSigned) | ||||||||
| 43 | Result = In1.sadd_ov(In2, Overflow); | ||||||||
| 44 | else | ||||||||
| 45 | Result = In1.uadd_ov(In2, Overflow); | ||||||||
| 46 | |||||||||
| 47 | return Overflow; | ||||||||
| 48 | } | ||||||||
| 49 | |||||||||
| 50 | /// Compute Result = In1-In2, returning true if the result overflowed for this | ||||||||
| 51 | /// type. | ||||||||
| 52 | static bool subWithOverflow(APInt &Result, const APInt &In1, | ||||||||
| 53 | const APInt &In2, bool IsSigned = false) { | ||||||||
| 54 | bool Overflow; | ||||||||
| 55 | if (IsSigned) | ||||||||
| 56 | Result = In1.ssub_ov(In2, Overflow); | ||||||||
| 57 | else | ||||||||
| 58 | Result = In1.usub_ov(In2, Overflow); | ||||||||
| 59 | |||||||||
| 60 | return Overflow; | ||||||||
| 61 | } | ||||||||
| 62 | |||||||||
| 63 | /// Given an icmp instruction, return true if any use of this comparison is a | ||||||||
| 64 | /// branch on sign bit comparison. | ||||||||
| 65 | static bool hasBranchUse(ICmpInst &I) { | ||||||||
| 66 | for (auto *U : I.users()) | ||||||||
| 67 | if (isa<BranchInst>(U)) | ||||||||
| 68 | return true; | ||||||||
| 69 | return false; | ||||||||
| 70 | } | ||||||||
| 71 | |||||||||
| 72 | /// Returns true if the exploded icmp can be expressed as a signed comparison | ||||||||
| 73 | /// to zero and updates the predicate accordingly. | ||||||||
| 74 | /// The signedness of the comparison is preserved. | ||||||||
| 75 | /// TODO: Refactor with decomposeBitTestICmp()? | ||||||||
| 76 | static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { | ||||||||
| 77 | if (!ICmpInst::isSigned(Pred)) | ||||||||
| 78 | return false; | ||||||||
| 79 | |||||||||
| 80 | if (C.isNullValue()) | ||||||||
| 81 | return ICmpInst::isRelational(Pred); | ||||||||
| 82 | |||||||||
| 83 | if (C.isOneValue()) { | ||||||||
| 84 | if (Pred == ICmpInst::ICMP_SLT) { | ||||||||
| 85 | Pred = ICmpInst::ICMP_SLE; | ||||||||
| 86 | return true; | ||||||||
| 87 | } | ||||||||
| 88 | } else if (C.isAllOnesValue()) { | ||||||||
| 89 | if (Pred == ICmpInst::ICMP_SGT) { | ||||||||
| 90 | Pred = ICmpInst::ICMP_SGE; | ||||||||
| 91 | return true; | ||||||||
| 92 | } | ||||||||
| 93 | } | ||||||||
| 94 | |||||||||
| 95 | return false; | ||||||||
| 96 | } | ||||||||
| 97 | |||||||||
| 98 | /// Given a signed integer type and a set of known zero and one bits, compute | ||||||||
| 99 | /// the maximum and minimum values that could have the specified known zero and | ||||||||
| 100 | /// known one bits, returning them in Min/Max. | ||||||||
| 101 | /// TODO: Move to method on KnownBits struct? | ||||||||
| 102 | static void computeSignedMinMaxValuesFromKnownBits(const KnownBits &Known, | ||||||||
| 103 | APInt &Min, APInt &Max) { | ||||||||
| 104 | assert(Known.getBitWidth() == Min.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 106, __PRETTY_FUNCTION__)) | ||||||||
| 105 | Known.getBitWidth() == Max.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 106, __PRETTY_FUNCTION__)) | ||||||||
| 106 | "KnownZero, KnownOne and Min, Max must have equal bitwidth.")((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 106, __PRETTY_FUNCTION__)); | ||||||||
| 107 | APInt UnknownBits = ~(Known.Zero|Known.One); | ||||||||
| 108 | |||||||||
| 109 | // The minimum value is when all unknown bits are zeros, EXCEPT for the sign | ||||||||
| 110 | // bit if it is unknown. | ||||||||
| 111 | Min = Known.One; | ||||||||
| 112 | Max = Known.One|UnknownBits; | ||||||||
| 113 | |||||||||
| 114 | if (UnknownBits.isNegative()) { // Sign bit is unknown | ||||||||
| 115 | Min.setSignBit(); | ||||||||
| 116 | Max.clearSignBit(); | ||||||||
| 117 | } | ||||||||
| 118 | } | ||||||||
| 119 | |||||||||
| 120 | /// Given an unsigned integer type and a set of known zero and one bits, compute | ||||||||
| 121 | /// the maximum and minimum values that could have the specified known zero and | ||||||||
| 122 | /// known one bits, returning them in Min/Max. | ||||||||
| 123 | /// TODO: Move to method on KnownBits struct? | ||||||||
| 124 | static void computeUnsignedMinMaxValuesFromKnownBits(const KnownBits &Known, | ||||||||
| 125 | APInt &Min, APInt &Max) { | ||||||||
| 126 | assert(Known.getBitWidth() == Min.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 128, __PRETTY_FUNCTION__)) | ||||||||
| 127 | Known.getBitWidth() == Max.getBitWidth() &&((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 128, __PRETTY_FUNCTION__)) | ||||||||
| 128 | "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.")((Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth () == Max.getBitWidth() && "Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth." ) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == Min.getBitWidth() && Known.getBitWidth() == Max.getBitWidth() && \"Ty, KnownZero, KnownOne and Min, Max must have equal bitwidth.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 128, __PRETTY_FUNCTION__)); | ||||||||
| 129 | APInt UnknownBits = ~(Known.Zero|Known.One); | ||||||||
| 130 | |||||||||
| 131 | // The minimum value is when the unknown bits are all zeros. | ||||||||
| 132 | Min = Known.One; | ||||||||
| 133 | // The maximum value is when the unknown bits are all ones. | ||||||||
| 134 | Max = Known.One|UnknownBits; | ||||||||
| 135 | } | ||||||||
| 136 | |||||||||
| 137 | /// This is called when we see this pattern: | ||||||||
| 138 | /// cmp pred (load (gep GV, ...)), cmpcst | ||||||||
| 139 | /// where GV is a global variable with a constant initializer. Try to simplify | ||||||||
| 140 | /// this into some simple computation that does not need the load. For example | ||||||||
| 141 | /// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3". | ||||||||
| 142 | /// | ||||||||
| 143 | /// If AndCst is non-null, then the loaded value is masked with that constant | ||||||||
| 144 | /// before doing the comparison. This handles cases like "A[i]&4 == 0". | ||||||||
| 145 | Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, | ||||||||
| 146 | GlobalVariable *GV, | ||||||||
| 147 | CmpInst &ICI, | ||||||||
| 148 | ConstantInt *AndCst) { | ||||||||
| 149 | Constant *Init = GV->getInitializer(); | ||||||||
| 150 | if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init)) | ||||||||
| 151 | return nullptr; | ||||||||
| 152 | |||||||||
| 153 | uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); | ||||||||
| 154 | // Don't blow up on huge arrays. | ||||||||
| 155 | if (ArrayElementCount > MaxArraySizeForCombine) | ||||||||
| 156 | return nullptr; | ||||||||
| 157 | |||||||||
| 158 | // There are many forms of this optimization we can handle, for now, just do | ||||||||
| 159 | // the simple index into a single-dimensional array. | ||||||||
| 160 | // | ||||||||
| 161 | // Require: GEP GV, 0, i {{, constant indices}} | ||||||||
| 162 | if (GEP->getNumOperands() < 3 || | ||||||||
| 163 | !isa<ConstantInt>(GEP->getOperand(1)) || | ||||||||
| 164 | !cast<ConstantInt>(GEP->getOperand(1))->isZero() || | ||||||||
| 165 | isa<Constant>(GEP->getOperand(2))) | ||||||||
| 166 | return nullptr; | ||||||||
| 167 | |||||||||
| 168 | // Check that indices after the variable are constants and in-range for the | ||||||||
| 169 | // type they index. Collect the indices. This is typically for arrays of | ||||||||
| 170 | // structs. | ||||||||
| 171 | SmallVector<unsigned, 4> LaterIndices; | ||||||||
| 172 | |||||||||
| 173 | Type *EltTy = Init->getType()->getArrayElementType(); | ||||||||
| 174 | for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) { | ||||||||
| 175 | ConstantInt *Idx = dyn_cast<ConstantInt>(GEP->getOperand(i)); | ||||||||
| 176 | if (!Idx) return nullptr; // Variable index. | ||||||||
| 177 | |||||||||
| 178 | uint64_t IdxVal = Idx->getZExtValue(); | ||||||||
| 179 | if ((unsigned)IdxVal != IdxVal) return nullptr; // Too large array index. | ||||||||
| 180 | |||||||||
| 181 | if (StructType *STy = dyn_cast<StructType>(EltTy)) | ||||||||
| 182 | EltTy = STy->getElementType(IdxVal); | ||||||||
| 183 | else if (ArrayType *ATy = dyn_cast<ArrayType>(EltTy)) { | ||||||||
| 184 | if (IdxVal >= ATy->getNumElements()) return nullptr; | ||||||||
| 185 | EltTy = ATy->getElementType(); | ||||||||
| 186 | } else { | ||||||||
| 187 | return nullptr; // Unknown type. | ||||||||
| 188 | } | ||||||||
| 189 | |||||||||
| 190 | LaterIndices.push_back(IdxVal); | ||||||||
| 191 | } | ||||||||
| 192 | |||||||||
| 193 | enum { Overdefined = -3, Undefined = -2 }; | ||||||||
| 194 | |||||||||
| 195 | // Variables for our state machines. | ||||||||
| 196 | |||||||||
| 197 | // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form | ||||||||
| 198 | // "i == 47 | i == 87", where 47 is the first index the condition is true for, | ||||||||
| 199 | // and 87 is the second (and last) index. FirstTrueElement is -2 when | ||||||||
| 200 | // undefined, otherwise set to the first true element. SecondTrueElement is | ||||||||
| 201 | // -2 when undefined, -3 when overdefined and >= 0 when that index is true. | ||||||||
| 202 | int FirstTrueElement = Undefined, SecondTrueElement = Undefined; | ||||||||
| 203 | |||||||||
| 204 | // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the | ||||||||
| 205 | // form "i != 47 & i != 87". Same state transitions as for true elements. | ||||||||
| 206 | int FirstFalseElement = Undefined, SecondFalseElement = Undefined; | ||||||||
| 207 | |||||||||
| 208 | /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these | ||||||||
| 209 | /// define a state machine that triggers for ranges of values that the index | ||||||||
| 210 | /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'. | ||||||||
| 211 | /// This is -2 when undefined, -3 when overdefined, and otherwise the last | ||||||||
| 212 | /// index in the range (inclusive). We use -2 for undefined here because we | ||||||||
| 213 | /// use relative comparisons and don't want 0-1 to match -1. | ||||||||
| 214 | int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined; | ||||||||
| 215 | |||||||||
| 216 | // MagicBitvector - This is a magic bitvector where we set a bit if the | ||||||||
| 217 | // comparison is true for element 'i'. If there are 64 elements or less in | ||||||||
| 218 | // the array, this will fully represent all the comparison results. | ||||||||
| 219 | uint64_t MagicBitvector = 0; | ||||||||
| 220 | |||||||||
| 221 | // Scan the array and see if one of our patterns matches. | ||||||||
| 222 | Constant *CompareRHS = cast<Constant>(ICI.getOperand(1)); | ||||||||
| 223 | for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) { | ||||||||
| 224 | Constant *Elt = Init->getAggregateElement(i); | ||||||||
| 225 | if (!Elt) return nullptr; | ||||||||
| 226 | |||||||||
| 227 | // If this is indexing an array of structures, get the structure element. | ||||||||
| 228 | if (!LaterIndices.empty()) | ||||||||
| 229 | Elt = ConstantExpr::getExtractValue(Elt, LaterIndices); | ||||||||
| 230 | |||||||||
| 231 | // If the element is masked, handle it. | ||||||||
| 232 | if (AndCst) Elt = ConstantExpr::getAnd(Elt, AndCst); | ||||||||
| 233 | |||||||||
| 234 | // Find out if the comparison would be true or false for the i'th element. | ||||||||
| 235 | Constant *C = ConstantFoldCompareInstOperands(ICI.getPredicate(), Elt, | ||||||||
| 236 | CompareRHS, DL, &TLI); | ||||||||
| 237 | // If the result is undef for this element, ignore it. | ||||||||
| 238 | if (isa<UndefValue>(C)) { | ||||||||
| 239 | // Extend range state machines to cover this element in case there is an | ||||||||
| 240 | // undef in the middle of the range. | ||||||||
| 241 | if (TrueRangeEnd == (int)i-1) | ||||||||
| 242 | TrueRangeEnd = i; | ||||||||
| 243 | if (FalseRangeEnd == (int)i-1) | ||||||||
| 244 | FalseRangeEnd = i; | ||||||||
| 245 | continue; | ||||||||
| 246 | } | ||||||||
| 247 | |||||||||
| 248 | // If we can't compute the result for any of the elements, we have to give | ||||||||
| 249 | // up evaluating the entire conditional. | ||||||||
| 250 | if (!isa<ConstantInt>(C)) return nullptr; | ||||||||
| 251 | |||||||||
| 252 | // Otherwise, we know if the comparison is true or false for this element, | ||||||||
| 253 | // update our state machines. | ||||||||
| 254 | bool IsTrueForElt = !cast<ConstantInt>(C)->isZero(); | ||||||||
| 255 | |||||||||
| 256 | // State machine for single/double/range index comparison. | ||||||||
| 257 | if (IsTrueForElt) { | ||||||||
| 258 | // Update the TrueElement state machine. | ||||||||
| 259 | if (FirstTrueElement == Undefined) | ||||||||
| 260 | FirstTrueElement = TrueRangeEnd = i; // First true element. | ||||||||
| 261 | else { | ||||||||
| 262 | // Update double-compare state machine. | ||||||||
| 263 | if (SecondTrueElement == Undefined) | ||||||||
| 264 | SecondTrueElement = i; | ||||||||
| 265 | else | ||||||||
| 266 | SecondTrueElement = Overdefined; | ||||||||
| 267 | |||||||||
| 268 | // Update range state machine. | ||||||||
| 269 | if (TrueRangeEnd == (int)i-1) | ||||||||
| 270 | TrueRangeEnd = i; | ||||||||
| 271 | else | ||||||||
| 272 | TrueRangeEnd = Overdefined; | ||||||||
| 273 | } | ||||||||
| 274 | } else { | ||||||||
| 275 | // Update the FalseElement state machine. | ||||||||
| 276 | if (FirstFalseElement == Undefined) | ||||||||
| 277 | FirstFalseElement = FalseRangeEnd = i; // First false element. | ||||||||
| 278 | else { | ||||||||
| 279 | // Update double-compare state machine. | ||||||||
| 280 | if (SecondFalseElement == Undefined) | ||||||||
| 281 | SecondFalseElement = i; | ||||||||
| 282 | else | ||||||||
| 283 | SecondFalseElement = Overdefined; | ||||||||
| 284 | |||||||||
| 285 | // Update range state machine. | ||||||||
| 286 | if (FalseRangeEnd == (int)i-1) | ||||||||
| 287 | FalseRangeEnd = i; | ||||||||
| 288 | else | ||||||||
| 289 | FalseRangeEnd = Overdefined; | ||||||||
| 290 | } | ||||||||
| 291 | } | ||||||||
| 292 | |||||||||
| 293 | // If this element is in range, update our magic bitvector. | ||||||||
| 294 | if (i < 64 && IsTrueForElt) | ||||||||
| 295 | MagicBitvector |= 1ULL << i; | ||||||||
| 296 | |||||||||
| 297 | // If all of our states become overdefined, bail out early. Since the | ||||||||
| 298 | // predicate is expensive, only check it every 8 elements. This is only | ||||||||
| 299 | // really useful for really huge arrays. | ||||||||
| 300 | if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined && | ||||||||
| 301 | SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined && | ||||||||
| 302 | FalseRangeEnd == Overdefined) | ||||||||
| 303 | return nullptr; | ||||||||
| 304 | } | ||||||||
| 305 | |||||||||
| 306 | // Now that we've scanned the entire array, emit our new comparison(s). We | ||||||||
| 307 | // order the state machines in complexity of the generated code. | ||||||||
| 308 | Value *Idx = GEP->getOperand(2); | ||||||||
| 309 | |||||||||
| 310 | // If the index is larger than the pointer size of the target, truncate the | ||||||||
| 311 | // index down like the GEP would do implicitly. We don't have to do this for | ||||||||
| 312 | // an inbounds GEP because the index can't be out of range. | ||||||||
| 313 | if (!GEP->isInBounds()) { | ||||||||
| 314 | Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); | ||||||||
| 315 | unsigned PtrSize = IntPtrTy->getIntegerBitWidth(); | ||||||||
| 316 | if (Idx->getType()->getPrimitiveSizeInBits() > PtrSize) | ||||||||
| 317 | Idx = Builder.CreateTrunc(Idx, IntPtrTy); | ||||||||
| 318 | } | ||||||||
| 319 | |||||||||
| 320 | // If the comparison is only true for one or two elements, emit direct | ||||||||
| 321 | // comparisons. | ||||||||
| 322 | if (SecondTrueElement != Overdefined) { | ||||||||
| 323 | // None true -> false. | ||||||||
| 324 | if (FirstTrueElement == Undefined) | ||||||||
| 325 | return replaceInstUsesWith(ICI, Builder.getFalse()); | ||||||||
| 326 | |||||||||
| 327 | Value *FirstTrueIdx = ConstantInt::get(Idx->getType(), FirstTrueElement); | ||||||||
| 328 | |||||||||
| 329 | // True for one element -> 'i == 47'. | ||||||||
| 330 | if (SecondTrueElement == Undefined) | ||||||||
| 331 | return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx); | ||||||||
| 332 | |||||||||
| 333 | // True for two elements -> 'i == 47 | i == 72'. | ||||||||
| 334 | Value *C1 = Builder.CreateICmpEQ(Idx, FirstTrueIdx); | ||||||||
| 335 | Value *SecondTrueIdx = ConstantInt::get(Idx->getType(), SecondTrueElement); | ||||||||
| 336 | Value *C2 = Builder.CreateICmpEQ(Idx, SecondTrueIdx); | ||||||||
| 337 | return BinaryOperator::CreateOr(C1, C2); | ||||||||
| 338 | } | ||||||||
| 339 | |||||||||
| 340 | // If the comparison is only false for one or two elements, emit direct | ||||||||
| 341 | // comparisons. | ||||||||
| 342 | if (SecondFalseElement != Overdefined) { | ||||||||
| 343 | // None false -> true. | ||||||||
| 344 | if (FirstFalseElement == Undefined) | ||||||||
| 345 | return replaceInstUsesWith(ICI, Builder.getTrue()); | ||||||||
| 346 | |||||||||
| 347 | Value *FirstFalseIdx = ConstantInt::get(Idx->getType(), FirstFalseElement); | ||||||||
| 348 | |||||||||
| 349 | // False for one element -> 'i != 47'. | ||||||||
| 350 | if (SecondFalseElement == Undefined) | ||||||||
| 351 | return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx); | ||||||||
| 352 | |||||||||
| 353 | // False for two elements -> 'i != 47 & i != 72'. | ||||||||
| 354 | Value *C1 = Builder.CreateICmpNE(Idx, FirstFalseIdx); | ||||||||
| 355 | Value *SecondFalseIdx = ConstantInt::get(Idx->getType(),SecondFalseElement); | ||||||||
| 356 | Value *C2 = Builder.CreateICmpNE(Idx, SecondFalseIdx); | ||||||||
| 357 | return BinaryOperator::CreateAnd(C1, C2); | ||||||||
| 358 | } | ||||||||
| 359 | |||||||||
| 360 | // If the comparison can be replaced with a range comparison for the elements | ||||||||
| 361 | // where it is true, emit the range check. | ||||||||
| 362 | if (TrueRangeEnd != Overdefined) { | ||||||||
| 363 | assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare")((TrueRangeEnd != FirstTrueElement && "Should emit single compare" ) ? static_cast<void> (0) : __assert_fail ("TrueRangeEnd != FirstTrueElement && \"Should emit single compare\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 363, __PRETTY_FUNCTION__)); | ||||||||
| 364 | |||||||||
| 365 | // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1). | ||||||||
| 366 | if (FirstTrueElement) { | ||||||||
| 367 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstTrueElement); | ||||||||
| 368 | Idx = Builder.CreateAdd(Idx, Offs); | ||||||||
| 369 | } | ||||||||
| 370 | |||||||||
| 371 | Value *End = ConstantInt::get(Idx->getType(), | ||||||||
| 372 | TrueRangeEnd-FirstTrueElement+1); | ||||||||
| 373 | return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End); | ||||||||
| 374 | } | ||||||||
| 375 | |||||||||
| 376 | // False range check. | ||||||||
| 377 | if (FalseRangeEnd != Overdefined) { | ||||||||
| 378 | assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare")((FalseRangeEnd != FirstFalseElement && "Should emit single compare" ) ? static_cast<void> (0) : __assert_fail ("FalseRangeEnd != FirstFalseElement && \"Should emit single compare\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 378, __PRETTY_FUNCTION__)); | ||||||||
| 379 | // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse). | ||||||||
| 380 | if (FirstFalseElement) { | ||||||||
| 381 | Value *Offs = ConstantInt::get(Idx->getType(), -FirstFalseElement); | ||||||||
| 382 | Idx = Builder.CreateAdd(Idx, Offs); | ||||||||
| 383 | } | ||||||||
| 384 | |||||||||
| 385 | Value *End = ConstantInt::get(Idx->getType(), | ||||||||
| 386 | FalseRangeEnd-FirstFalseElement); | ||||||||
| 387 | return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End); | ||||||||
| 388 | } | ||||||||
| 389 | |||||||||
| 390 | // If a magic bitvector captures the entire comparison state | ||||||||
| 391 | // of this load, replace it with computation that does: | ||||||||
| 392 | // ((magic_cst >> i) & 1) != 0 | ||||||||
| 393 | { | ||||||||
| 394 | Type *Ty = nullptr; | ||||||||
| 395 | |||||||||
| 396 | // Look for an appropriate type: | ||||||||
| 397 | // - The type of Idx if the magic fits | ||||||||
| 398 | // - The smallest fitting legal type | ||||||||
| 399 | if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth()) | ||||||||
| 400 | Ty = Idx->getType(); | ||||||||
| 401 | else | ||||||||
| 402 | Ty = DL.getSmallestLegalIntType(Init->getContext(), ArrayElementCount); | ||||||||
| 403 | |||||||||
| 404 | if (Ty) { | ||||||||
| 405 | Value *V = Builder.CreateIntCast(Idx, Ty, false); | ||||||||
| 406 | V = Builder.CreateLShr(ConstantInt::get(Ty, MagicBitvector), V); | ||||||||
| 407 | V = Builder.CreateAnd(ConstantInt::get(Ty, 1), V); | ||||||||
| 408 | return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, 0)); | ||||||||
| 409 | } | ||||||||
| 410 | } | ||||||||
| 411 | |||||||||
| 412 | return nullptr; | ||||||||
| 413 | } | ||||||||
| 414 | |||||||||
| 415 | /// Return a value that can be used to compare the *offset* implied by a GEP to | ||||||||
| 416 | /// zero. For example, if we have &A[i], we want to return 'i' for | ||||||||
| 417 | /// "icmp ne i, 0". Note that, in general, indices can be complex, and scales | ||||||||
| 418 | /// are involved. The above expression would also be legal to codegen as | ||||||||
| 419 | /// "icmp ne (i*4), 0" (assuming A is a pointer to i32). | ||||||||
| 420 | /// This latter form is less amenable to optimization though, and we are allowed | ||||||||
| 421 | /// to generate the first by knowing that pointer arithmetic doesn't overflow. | ||||||||
| 422 | /// | ||||||||
| 423 | /// If we can't emit an optimized form for this expression, this returns null. | ||||||||
| 424 | /// | ||||||||
| 425 | static Value *evaluateGEPOffsetExpression(User *GEP, InstCombiner &IC, | ||||||||
| 426 | const DataLayout &DL) { | ||||||||
| 427 | gep_type_iterator GTI = gep_type_begin(GEP); | ||||||||
| 428 | |||||||||
| 429 | // Check to see if this gep only has a single variable index. If so, and if | ||||||||
| 430 | // any constant indices are a multiple of its scale, then we can compute this | ||||||||
| 431 | // in terms of the scale of the variable index. For example, if the GEP | ||||||||
| 432 | // implies an offset of "12 + i*4", then we can codegen this as "3 + i", | ||||||||
| 433 | // because the expression will cross zero at the same point. | ||||||||
| 434 | unsigned i, e = GEP->getNumOperands(); | ||||||||
| 435 | int64_t Offset = 0; | ||||||||
| 436 | for (i = 1; i != e; ++i, ++GTI) { | ||||||||
| 437 | if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { | ||||||||
| 438 | // Compute the aggregate offset of constant indices. | ||||||||
| 439 | if (CI->isZero()) continue; | ||||||||
| 440 | |||||||||
| 441 | // Handle a struct index, which adds its field offset to the pointer. | ||||||||
| 442 | if (StructType *STy = GTI.getStructTypeOrNull()) { | ||||||||
| 443 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | ||||||||
| 444 | } else { | ||||||||
| 445 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||||||
| 446 | Offset += Size*CI->getSExtValue(); | ||||||||
| 447 | } | ||||||||
| 448 | } else { | ||||||||
| 449 | // Found our variable index. | ||||||||
| 450 | break; | ||||||||
| 451 | } | ||||||||
| 452 | } | ||||||||
| 453 | |||||||||
| 454 | // If there are no variable indices, we must have a constant offset, just | ||||||||
| 455 | // evaluate it the general way. | ||||||||
| 456 | if (i == e) return nullptr; | ||||||||
| 457 | |||||||||
| 458 | Value *VariableIdx = GEP->getOperand(i); | ||||||||
| 459 | // Determine the scale factor of the variable element. For example, this is | ||||||||
| 460 | // 4 if the variable index is into an array of i32. | ||||||||
| 461 | uint64_t VariableScale = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||||||
| 462 | |||||||||
| 463 | // Verify that there are no other variable indices. If so, emit the hard way. | ||||||||
| 464 | for (++i, ++GTI; i != e; ++i, ++GTI) { | ||||||||
| 465 | ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i)); | ||||||||
| 466 | if (!CI) return nullptr; | ||||||||
| 467 | |||||||||
| 468 | // Compute the aggregate offset of constant indices. | ||||||||
| 469 | if (CI->isZero()) continue; | ||||||||
| 470 | |||||||||
| 471 | // Handle a struct index, which adds its field offset to the pointer. | ||||||||
| 472 | if (StructType *STy = GTI.getStructTypeOrNull()) { | ||||||||
| 473 | Offset += DL.getStructLayout(STy)->getElementOffset(CI->getZExtValue()); | ||||||||
| 474 | } else { | ||||||||
| 475 | uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()); | ||||||||
| 476 | Offset += Size*CI->getSExtValue(); | ||||||||
| 477 | } | ||||||||
| 478 | } | ||||||||
| 479 | |||||||||
| 480 | // Okay, we know we have a single variable index, which must be a | ||||||||
| 481 | // pointer/array/vector index. If there is no offset, life is simple, return | ||||||||
| 482 | // the index. | ||||||||
| 483 | Type *IntPtrTy = DL.getIntPtrType(GEP->getOperand(0)->getType()); | ||||||||
| 484 | unsigned IntPtrWidth = IntPtrTy->getIntegerBitWidth(); | ||||||||
| 485 | if (Offset == 0) { | ||||||||
| 486 | // Cast to intptrty in case a truncation occurs. If an extension is needed, | ||||||||
| 487 | // we don't need to bother extending: the extension won't affect where the | ||||||||
| 488 | // computation crosses zero. | ||||||||
| 489 | if (VariableIdx->getType()->getPrimitiveSizeInBits() > IntPtrWidth) { | ||||||||
| 490 | VariableIdx = IC.Builder.CreateTrunc(VariableIdx, IntPtrTy); | ||||||||
| 491 | } | ||||||||
| 492 | return VariableIdx; | ||||||||
| 493 | } | ||||||||
| 494 | |||||||||
| 495 | // Otherwise, there is an index. The computation we will do will be modulo | ||||||||
| 496 | // the pointer size. | ||||||||
| 497 | Offset = SignExtend64(Offset, IntPtrWidth); | ||||||||
| 498 | VariableScale = SignExtend64(VariableScale, IntPtrWidth); | ||||||||
| 499 | |||||||||
| 500 | // To do this transformation, any constant index must be a multiple of the | ||||||||
| 501 | // variable scale factor. For example, we can evaluate "12 + 4*i" as "3 + i", | ||||||||
| 502 | // but we can't evaluate "10 + 3*i" in terms of i. Check that the offset is a | ||||||||
| 503 | // multiple of the variable scale. | ||||||||
| 504 | int64_t NewOffs = Offset / (int64_t)VariableScale; | ||||||||
| 505 | if (Offset != NewOffs*(int64_t)VariableScale) | ||||||||
| 506 | return nullptr; | ||||||||
| 507 | |||||||||
| 508 | // Okay, we can do this evaluation. Start by converting the index to intptr. | ||||||||
| 509 | if (VariableIdx->getType() != IntPtrTy) | ||||||||
| 510 | VariableIdx = IC.Builder.CreateIntCast(VariableIdx, IntPtrTy, | ||||||||
| 511 | true /*Signed*/); | ||||||||
| 512 | Constant *OffsetVal = ConstantInt::get(IntPtrTy, NewOffs); | ||||||||
| 513 | return IC.Builder.CreateAdd(VariableIdx, OffsetVal, "offset"); | ||||||||
| 514 | } | ||||||||
| 515 | |||||||||
| 516 | /// Returns true if we can rewrite Start as a GEP with pointer Base | ||||||||
| 517 | /// and some integer offset. The nodes that need to be re-written | ||||||||
| 518 | /// for this transformation will be added to Explored. | ||||||||
| 519 | static bool canRewriteGEPAsOffset(Value *Start, Value *Base, | ||||||||
| 520 | const DataLayout &DL, | ||||||||
| 521 | SetVector<Value *> &Explored) { | ||||||||
| 522 | SmallVector<Value *, 16> WorkList(1, Start); | ||||||||
| 523 | Explored.insert(Base); | ||||||||
| 524 | |||||||||
| 525 | // The following traversal gives us an order which can be used | ||||||||
| 526 | // when doing the final transformation. Since in the final | ||||||||
| 527 | // transformation we create the PHI replacement instructions first, | ||||||||
| 528 | // we don't have to get them in any particular order. | ||||||||
| 529 | // | ||||||||
| 530 | // However, for other instructions we will have to traverse the | ||||||||
| 531 | // operands of an instruction first, which means that we have to | ||||||||
| 532 | // do a post-order traversal. | ||||||||
| 533 | while (!WorkList.empty()) { | ||||||||
| 534 | SetVector<PHINode *> PHIs; | ||||||||
| 535 | |||||||||
| 536 | while (!WorkList.empty()) { | ||||||||
| 537 | if (Explored.size() >= 100) | ||||||||
| 538 | return false; | ||||||||
| 539 | |||||||||
| 540 | Value *V = WorkList.back(); | ||||||||
| 541 | |||||||||
| 542 | if (Explored.count(V) != 0) { | ||||||||
| 543 | WorkList.pop_back(); | ||||||||
| 544 | continue; | ||||||||
| 545 | } | ||||||||
| 546 | |||||||||
| 547 | if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) && | ||||||||
| 548 | !isa<GetElementPtrInst>(V) && !isa<PHINode>(V)) | ||||||||
| 549 | // We've found some value that we can't explore which is different from | ||||||||
| 550 | // the base. Therefore we can't do this transformation. | ||||||||
| 551 | return false; | ||||||||
| 552 | |||||||||
| 553 | if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) { | ||||||||
| 554 | auto *CI = dyn_cast<CastInst>(V); | ||||||||
| 555 | if (!CI->isNoopCast(DL)) | ||||||||
| 556 | return false; | ||||||||
| 557 | |||||||||
| 558 | if (Explored.count(CI->getOperand(0)) == 0) | ||||||||
| 559 | WorkList.push_back(CI->getOperand(0)); | ||||||||
| 560 | } | ||||||||
| 561 | |||||||||
| 562 | if (auto *GEP = dyn_cast<GEPOperator>(V)) { | ||||||||
| 563 | // We're limiting the GEP to having one index. This will preserve | ||||||||
| 564 | // the original pointer type. We could handle more cases in the | ||||||||
| 565 | // future. | ||||||||
| 566 | if (GEP->getNumIndices() != 1 || !GEP->isInBounds() || | ||||||||
| 567 | GEP->getType() != Start->getType()) | ||||||||
| 568 | return false; | ||||||||
| 569 | |||||||||
| 570 | if (Explored.count(GEP->getOperand(0)) == 0) | ||||||||
| 571 | WorkList.push_back(GEP->getOperand(0)); | ||||||||
| 572 | } | ||||||||
| 573 | |||||||||
| 574 | if (WorkList.back() == V) { | ||||||||
| 575 | WorkList.pop_back(); | ||||||||
| 576 | // We've finished visiting this node, mark it as such. | ||||||||
| 577 | Explored.insert(V); | ||||||||
| 578 | } | ||||||||
| 579 | |||||||||
| 580 | if (auto *PN = dyn_cast<PHINode>(V)) { | ||||||||
| 581 | // We cannot transform PHIs on unsplittable basic blocks. | ||||||||
| 582 | if (isa<CatchSwitchInst>(PN->getParent()->getTerminator())) | ||||||||
| 583 | return false; | ||||||||
| 584 | Explored.insert(PN); | ||||||||
| 585 | PHIs.insert(PN); | ||||||||
| 586 | } | ||||||||
| 587 | } | ||||||||
| 588 | |||||||||
| 589 | // Explore the PHI nodes further. | ||||||||
| 590 | for (auto *PN : PHIs) | ||||||||
| 591 | for (Value *Op : PN->incoming_values()) | ||||||||
| 592 | if (Explored.count(Op) == 0) | ||||||||
| 593 | WorkList.push_back(Op); | ||||||||
| 594 | } | ||||||||
| 595 | |||||||||
| 596 | // Make sure that we can do this. Since we can't insert GEPs in a basic | ||||||||
| 597 | // block before a PHI node, we can't easily do this transformation if | ||||||||
| 598 | // we have PHI node users of transformed instructions. | ||||||||
| 599 | for (Value *Val : Explored) { | ||||||||
| 600 | for (Value *Use : Val->uses()) { | ||||||||
| 601 | |||||||||
| 602 | auto *PHI = dyn_cast<PHINode>(Use); | ||||||||
| 603 | auto *Inst = dyn_cast<Instruction>(Val); | ||||||||
| 604 | |||||||||
| 605 | if (Inst == Base || Inst == PHI || !Inst || !PHI || | ||||||||
| 606 | Explored.count(PHI) == 0) | ||||||||
| 607 | continue; | ||||||||
| 608 | |||||||||
| 609 | if (PHI->getParent() == Inst->getParent()) | ||||||||
| 610 | return false; | ||||||||
| 611 | } | ||||||||
| 612 | } | ||||||||
| 613 | return true; | ||||||||
| 614 | } | ||||||||
| 615 | |||||||||
| 616 | // Sets the appropriate insert point on Builder where we can add | ||||||||
| 617 | // a replacement Instruction for V (if that is possible). | ||||||||
| 618 | static void setInsertionPoint(IRBuilder<> &Builder, Value *V, | ||||||||
| 619 | bool Before = true) { | ||||||||
| 620 | if (auto *PHI = dyn_cast<PHINode>(V)) { | ||||||||
| 621 | Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt()); | ||||||||
| 622 | return; | ||||||||
| 623 | } | ||||||||
| 624 | if (auto *I = dyn_cast<Instruction>(V)) { | ||||||||
| 625 | if (!Before) | ||||||||
| 626 | I = &*std::next(I->getIterator()); | ||||||||
| 627 | Builder.SetInsertPoint(I); | ||||||||
| 628 | return; | ||||||||
| 629 | } | ||||||||
| 630 | if (auto *A = dyn_cast<Argument>(V)) { | ||||||||
| 631 | // Set the insertion point in the entry block. | ||||||||
| 632 | BasicBlock &Entry = A->getParent()->getEntryBlock(); | ||||||||
| 633 | Builder.SetInsertPoint(&*Entry.getFirstInsertionPt()); | ||||||||
| 634 | return; | ||||||||
| 635 | } | ||||||||
| 636 | // Otherwise, this is a constant and we don't need to set a new | ||||||||
| 637 | // insertion point. | ||||||||
| 638 | assert(isa<Constant>(V) && "Setting insertion point for unknown value!")((isa<Constant>(V) && "Setting insertion point for unknown value!" ) ? static_cast<void> (0) : __assert_fail ("isa<Constant>(V) && \"Setting insertion point for unknown value!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 638, __PRETTY_FUNCTION__)); | ||||||||
| 639 | } | ||||||||
| 640 | |||||||||
| 641 | /// Returns a re-written value of Start as an indexed GEP using Base as a | ||||||||
| 642 | /// pointer. | ||||||||
| 643 | static Value *rewriteGEPAsOffset(Value *Start, Value *Base, | ||||||||
| 644 | const DataLayout &DL, | ||||||||
| 645 | SetVector<Value *> &Explored) { | ||||||||
| 646 | // Perform all the substitutions. This is a bit tricky because we can | ||||||||
| 647 | // have cycles in our use-def chains. | ||||||||
| 648 | // 1. Create the PHI nodes without any incoming values. | ||||||||
| 649 | // 2. Create all the other values. | ||||||||
| 650 | // 3. Add the edges for the PHI nodes. | ||||||||
| 651 | // 4. Emit GEPs to get the original pointers. | ||||||||
| 652 | // 5. Remove the original instructions. | ||||||||
| 653 | Type *IndexType = IntegerType::get( | ||||||||
| 654 | Base->getContext(), DL.getIndexTypeSizeInBits(Start->getType())); | ||||||||
| 655 | |||||||||
| 656 | DenseMap<Value *, Value *> NewInsts; | ||||||||
| 657 | NewInsts[Base] = ConstantInt::getNullValue(IndexType); | ||||||||
| 658 | |||||||||
| 659 | // Create the new PHI nodes, without adding any incoming values. | ||||||||
| 660 | for (Value *Val : Explored) { | ||||||||
| 661 | if (Val == Base) | ||||||||
| 662 | continue; | ||||||||
| 663 | // Create empty phi nodes. This avoids cyclic dependencies when creating | ||||||||
| 664 | // the remaining instructions. | ||||||||
| 665 | if (auto *PHI = dyn_cast<PHINode>(Val)) | ||||||||
| 666 | NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), | ||||||||
| 667 | PHI->getName() + ".idx", PHI); | ||||||||
| 668 | } | ||||||||
| 669 | IRBuilder<> Builder(Base->getContext()); | ||||||||
| 670 | |||||||||
| 671 | // Create all the other instructions. | ||||||||
| 672 | for (Value *Val : Explored) { | ||||||||
| 673 | |||||||||
| 674 | if (NewInsts.find(Val) != NewInsts.end()) | ||||||||
| 675 | continue; | ||||||||
| 676 | |||||||||
| 677 | if (auto *CI = dyn_cast<CastInst>(Val)) { | ||||||||
| 678 | // Don't get rid of the intermediate variable here; the store can grow | ||||||||
| 679 | // the map which will invalidate the reference to the input value. | ||||||||
| 680 | Value *V = NewInsts[CI->getOperand(0)]; | ||||||||
| 681 | NewInsts[CI] = V; | ||||||||
| 682 | continue; | ||||||||
| 683 | } | ||||||||
| 684 | if (auto *GEP = dyn_cast<GEPOperator>(Val)) { | ||||||||
| 685 | Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)] | ||||||||
| 686 | : GEP->getOperand(1); | ||||||||
| 687 | setInsertionPoint(Builder, GEP); | ||||||||
| 688 | // Indices might need to be sign extended. GEPs will magically do | ||||||||
| 689 | // this, but we need to do it ourselves here. | ||||||||
| 690 | if (Index->getType()->getScalarSizeInBits() != | ||||||||
| 691 | NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) { | ||||||||
| 692 | Index = Builder.CreateSExtOrTrunc( | ||||||||
| 693 | Index, NewInsts[GEP->getOperand(0)]->getType(), | ||||||||
| 694 | GEP->getOperand(0)->getName() + ".sext"); | ||||||||
| 695 | } | ||||||||
| 696 | |||||||||
| 697 | auto *Op = NewInsts[GEP->getOperand(0)]; | ||||||||
| 698 | if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero()) | ||||||||
| 699 | NewInsts[GEP] = Index; | ||||||||
| 700 | else | ||||||||
| 701 | NewInsts[GEP] = Builder.CreateNSWAdd( | ||||||||
| 702 | Op, Index, GEP->getOperand(0)->getName() + ".add"); | ||||||||
| 703 | continue; | ||||||||
| 704 | } | ||||||||
| 705 | if (isa<PHINode>(Val)) | ||||||||
| 706 | continue; | ||||||||
| 707 | |||||||||
| 708 | llvm_unreachable("Unexpected instruction type")::llvm::llvm_unreachable_internal("Unexpected instruction type" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 708); | ||||||||
| 709 | } | ||||||||
| 710 | |||||||||
| 711 | // Add the incoming values to the PHI nodes. | ||||||||
| 712 | for (Value *Val : Explored) { | ||||||||
| 713 | if (Val == Base) | ||||||||
| 714 | continue; | ||||||||
| 715 | // All the instructions have been created, we can now add edges to the | ||||||||
| 716 | // phi nodes. | ||||||||
| 717 | if (auto *PHI = dyn_cast<PHINode>(Val)) { | ||||||||
| 718 | PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); | ||||||||
| 719 | for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { | ||||||||
| 720 | Value *NewIncoming = PHI->getIncomingValue(I); | ||||||||
| 721 | |||||||||
| 722 | if (NewInsts.find(NewIncoming) != NewInsts.end()) | ||||||||
| 723 | NewIncoming = NewInsts[NewIncoming]; | ||||||||
| 724 | |||||||||
| 725 | NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I)); | ||||||||
| 726 | } | ||||||||
| 727 | } | ||||||||
| 728 | } | ||||||||
| 729 | |||||||||
| 730 | for (Value *Val : Explored) { | ||||||||
| 731 | if (Val == Base) | ||||||||
| 732 | continue; | ||||||||
| 733 | |||||||||
| 734 | // Depending on the type, for external users we have to emit | ||||||||
| 735 | // a GEP or a GEP + ptrtoint. | ||||||||
| 736 | setInsertionPoint(Builder, Val, false); | ||||||||
| 737 | |||||||||
| 738 | // If required, create an inttoptr instruction for Base. | ||||||||
| 739 | Value *NewBase = Base; | ||||||||
| 740 | if (!Base->getType()->isPointerTy()) | ||||||||
| 741 | NewBase = Builder.CreateBitOrPointerCast(Base, Start->getType(), | ||||||||
| 742 | Start->getName() + "to.ptr"); | ||||||||
| 743 | |||||||||
| 744 | Value *GEP = Builder.CreateInBoundsGEP( | ||||||||
| 745 | Start->getType()->getPointerElementType(), NewBase, | ||||||||
| 746 | makeArrayRef(NewInsts[Val]), Val->getName() + ".ptr"); | ||||||||
| 747 | |||||||||
| 748 | if (!Val->getType()->isPointerTy()) { | ||||||||
| 749 | Value *Cast = Builder.CreatePointerCast(GEP, Val->getType(), | ||||||||
| 750 | Val->getName() + ".conv"); | ||||||||
| 751 | GEP = Cast; | ||||||||
| 752 | } | ||||||||
| 753 | Val->replaceAllUsesWith(GEP); | ||||||||
| 754 | } | ||||||||
| 755 | |||||||||
| 756 | return NewInsts[Start]; | ||||||||
| 757 | } | ||||||||
| 758 | |||||||||
| 759 | /// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express | ||||||||
| 760 | /// the input Value as a constant indexed GEP. Returns a pair containing | ||||||||
| 761 | /// the GEPs Pointer and Index. | ||||||||
| 762 | static std::pair<Value *, Value *> | ||||||||
| 763 | getAsConstantIndexedAddress(Value *V, const DataLayout &DL) { | ||||||||
| 764 | Type *IndexType = IntegerType::get(V->getContext(), | ||||||||
| 765 | DL.getIndexTypeSizeInBits(V->getType())); | ||||||||
| 766 | |||||||||
| 767 | Constant *Index = ConstantInt::getNullValue(IndexType); | ||||||||
| 768 | while (true) { | ||||||||
| 769 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { | ||||||||
| 770 | // We accept only inbouds GEPs here to exclude the possibility of | ||||||||
| 771 | // overflow. | ||||||||
| 772 | if (!GEP->isInBounds()) | ||||||||
| 773 | break; | ||||||||
| 774 | if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 && | ||||||||
| 775 | GEP->getType() == V->getType()) { | ||||||||
| 776 | V = GEP->getOperand(0); | ||||||||
| 777 | Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1)); | ||||||||
| 778 | Index = ConstantExpr::getAdd( | ||||||||
| 779 | Index, ConstantExpr::getSExtOrBitCast(GEPIndex, IndexType)); | ||||||||
| 780 | continue; | ||||||||
| 781 | } | ||||||||
| 782 | break; | ||||||||
| 783 | } | ||||||||
| 784 | if (auto *CI = dyn_cast<IntToPtrInst>(V)) { | ||||||||
| 785 | if (!CI->isNoopCast(DL)) | ||||||||
| 786 | break; | ||||||||
| 787 | V = CI->getOperand(0); | ||||||||
| 788 | continue; | ||||||||
| 789 | } | ||||||||
| 790 | if (auto *CI = dyn_cast<PtrToIntInst>(V)) { | ||||||||
| 791 | if (!CI->isNoopCast(DL)) | ||||||||
| 792 | break; | ||||||||
| 793 | V = CI->getOperand(0); | ||||||||
| 794 | continue; | ||||||||
| 795 | } | ||||||||
| 796 | break; | ||||||||
| 797 | } | ||||||||
| 798 | return {V, Index}; | ||||||||
| 799 | } | ||||||||
| 800 | |||||||||
| 801 | /// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. | ||||||||
| 802 | /// We can look through PHIs, GEPs and casts in order to determine a common base | ||||||||
| 803 | /// between GEPLHS and RHS. | ||||||||
| 804 | static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, | ||||||||
| 805 | ICmpInst::Predicate Cond, | ||||||||
| 806 | const DataLayout &DL) { | ||||||||
| 807 | // FIXME: Support vector of pointers. | ||||||||
| 808 | if (GEPLHS->getType()->isVectorTy()) | ||||||||
| 809 | return nullptr; | ||||||||
| 810 | |||||||||
| 811 | if (!GEPLHS->hasAllConstantIndices()) | ||||||||
| 812 | return nullptr; | ||||||||
| 813 | |||||||||
| 814 | // Make sure the pointers have the same type. | ||||||||
| 815 | if (GEPLHS->getType() != RHS->getType()) | ||||||||
| 816 | return nullptr; | ||||||||
| 817 | |||||||||
| 818 | Value *PtrBase, *Index; | ||||||||
| 819 | std::tie(PtrBase, Index) = getAsConstantIndexedAddress(GEPLHS, DL); | ||||||||
| 820 | |||||||||
| 821 | // The set of nodes that will take part in this transformation. | ||||||||
| 822 | SetVector<Value *> Nodes; | ||||||||
| 823 | |||||||||
| 824 | if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes)) | ||||||||
| 825 | return nullptr; | ||||||||
| 826 | |||||||||
| 827 | // We know we can re-write this as | ||||||||
| 828 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) | ||||||||
| 829 | // Since we've only looked through inbouds GEPs we know that we | ||||||||
| 830 | // can't have overflow on either side. We can therefore re-write | ||||||||
| 831 | // this as: | ||||||||
| 832 | // OFFSET1 cmp OFFSET2 | ||||||||
| 833 | Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes); | ||||||||
| 834 | |||||||||
| 835 | // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written | ||||||||
| 836 | // GEP having PtrBase as the pointer base, and has returned in NewRHS the | ||||||||
| 837 | // offset. Since Index is the offset of LHS to the base pointer, we will now | ||||||||
| 838 | // compare the offsets instead of comparing the pointers. | ||||||||
| 839 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS); | ||||||||
| 840 | } | ||||||||
| 841 | |||||||||
| 842 | /// Fold comparisons between a GEP instruction and something else. At this point | ||||||||
| 843 | /// we know that the GEP is on the LHS of the comparison. | ||||||||
| 844 | Instruction *InstCombiner::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, | ||||||||
| 845 | ICmpInst::Predicate Cond, | ||||||||
| 846 | Instruction &I) { | ||||||||
| 847 | // Don't transform signed compares of GEPs into index compares. Even if the | ||||||||
| 848 | // GEP is inbounds, the final add of the base pointer can have signed overflow | ||||||||
| 849 | // and would change the result of the icmp. | ||||||||
| 850 | // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be | ||||||||
| 851 | // the maximum signed value for the pointer type. | ||||||||
| 852 | if (ICmpInst::isSigned(Cond)) | ||||||||
| 853 | return nullptr; | ||||||||
| 854 | |||||||||
| 855 | // Look through bitcasts and addrspacecasts. We do not however want to remove | ||||||||
| 856 | // 0 GEPs. | ||||||||
| 857 | if (!isa<GetElementPtrInst>(RHS)) | ||||||||
| 858 | RHS = RHS->stripPointerCasts(); | ||||||||
| 859 | |||||||||
| 860 | Value *PtrBase = GEPLHS->getOperand(0); | ||||||||
| 861 | // FIXME: Support vector pointer GEPs. | ||||||||
| 862 | if (PtrBase == RHS && GEPLHS->isInBounds() && | ||||||||
| 863 | !GEPLHS->getType()->isVectorTy()) { | ||||||||
| 864 | // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0). | ||||||||
| 865 | // This transformation (ignoring the base and scales) is valid because we | ||||||||
| 866 | // know pointers can't overflow since the gep is inbounds. See if we can | ||||||||
| 867 | // output an optimized form. | ||||||||
| 868 | Value *Offset = evaluateGEPOffsetExpression(GEPLHS, *this, DL); | ||||||||
| 869 | |||||||||
| 870 | // If not, synthesize the offset the hard way. | ||||||||
| 871 | if (!Offset) | ||||||||
| 872 | Offset = EmitGEPOffset(GEPLHS); | ||||||||
| 873 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Offset, | ||||||||
| 874 | Constant::getNullValue(Offset->getType())); | ||||||||
| 875 | } | ||||||||
| 876 | |||||||||
| 877 | if (GEPLHS->isInBounds() && ICmpInst::isEquality(Cond) && | ||||||||
| 878 | isa<Constant>(RHS) && cast<Constant>(RHS)->isNullValue() && | ||||||||
| 879 | !NullPointerIsDefined(I.getFunction(), | ||||||||
| 880 | RHS->getType()->getPointerAddressSpace())) { | ||||||||
| 881 | // For most address spaces, an allocation can't be placed at null, but null | ||||||||
| 882 | // itself is treated as a 0 size allocation in the in bounds rules. Thus, | ||||||||
| 883 | // the only valid inbounds address derived from null, is null itself. | ||||||||
| 884 | // Thus, we have four cases to consider: | ||||||||
| 885 | // 1) Base == nullptr, Offset == 0 -> inbounds, null | ||||||||
| 886 | // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds | ||||||||
| 887 | // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations) | ||||||||
| 888 | // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison) | ||||||||
| 889 | // | ||||||||
| 890 | // (Note if we're indexing a type of size 0, that simply collapses into one | ||||||||
| 891 | // of the buckets above.) | ||||||||
| 892 | // | ||||||||
| 893 | // In general, we're allowed to make values less poison (i.e. remove | ||||||||
| 894 | // sources of full UB), so in this case, we just select between the two | ||||||||
| 895 | // non-poison cases (1 and 4 above). | ||||||||
| 896 | // | ||||||||
| 897 | // For vectors, we apply the same reasoning on a per-lane basis. | ||||||||
| 898 | auto *Base = GEPLHS->getPointerOperand(); | ||||||||
| 899 | if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) { | ||||||||
| 900 | int NumElts = GEPLHS->getType()->getVectorNumElements(); | ||||||||
| 901 | Base = Builder.CreateVectorSplat(NumElts, Base); | ||||||||
| 902 | } | ||||||||
| 903 | return new ICmpInst(Cond, Base, | ||||||||
| 904 | ConstantExpr::getPointerBitCastOrAddrSpaceCast( | ||||||||
| 905 | cast<Constant>(RHS), Base->getType())); | ||||||||
| 906 | } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(RHS)) { | ||||||||
| 907 | // If the base pointers are different, but the indices are the same, just | ||||||||
| 908 | // compare the base pointer. | ||||||||
| 909 | if (PtrBase != GEPRHS->getOperand(0)) { | ||||||||
| 910 | bool IndicesTheSame = GEPLHS->getNumOperands()==GEPRHS->getNumOperands(); | ||||||||
| 911 | IndicesTheSame &= GEPLHS->getOperand(0)->getType() == | ||||||||
| 912 | GEPRHS->getOperand(0)->getType(); | ||||||||
| 913 | if (IndicesTheSame) | ||||||||
| 914 | for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i) | ||||||||
| 915 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | ||||||||
| 916 | IndicesTheSame = false; | ||||||||
| 917 | break; | ||||||||
| 918 | } | ||||||||
| 919 | |||||||||
| 920 | // If all indices are the same, just compare the base pointers. | ||||||||
| 921 | Type *BaseType = GEPLHS->getOperand(0)->getType(); | ||||||||
| 922 | if (IndicesTheSame && CmpInst::makeCmpResultType(BaseType) == I.getType()) | ||||||||
| 923 | return new ICmpInst(Cond, GEPLHS->getOperand(0), GEPRHS->getOperand(0)); | ||||||||
| 924 | |||||||||
| 925 | // If we're comparing GEPs with two base pointers that only differ in type | ||||||||
| 926 | // and both GEPs have only constant indices or just one use, then fold | ||||||||
| 927 | // the compare with the adjusted indices. | ||||||||
| 928 | // FIXME: Support vector of pointers. | ||||||||
| 929 | if (GEPLHS->isInBounds() && GEPRHS->isInBounds() && | ||||||||
| 930 | (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && | ||||||||
| 931 | (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) && | ||||||||
| 932 | PtrBase->stripPointerCasts() == | ||||||||
| 933 | GEPRHS->getOperand(0)->stripPointerCasts() && | ||||||||
| 934 | !GEPLHS->getType()->isVectorTy()) { | ||||||||
| 935 | Value *LOffset = EmitGEPOffset(GEPLHS); | ||||||||
| 936 | Value *ROffset = EmitGEPOffset(GEPRHS); | ||||||||
| 937 | |||||||||
| 938 | // If we looked through an addrspacecast between different sized address | ||||||||
| 939 | // spaces, the LHS and RHS pointers are different sized | ||||||||
| 940 | // integers. Truncate to the smaller one. | ||||||||
| 941 | Type *LHSIndexTy = LOffset->getType(); | ||||||||
| 942 | Type *RHSIndexTy = ROffset->getType(); | ||||||||
| 943 | if (LHSIndexTy != RHSIndexTy) { | ||||||||
| 944 | if (LHSIndexTy->getPrimitiveSizeInBits() < | ||||||||
| 945 | RHSIndexTy->getPrimitiveSizeInBits()) { | ||||||||
| 946 | ROffset = Builder.CreateTrunc(ROffset, LHSIndexTy); | ||||||||
| 947 | } else | ||||||||
| 948 | LOffset = Builder.CreateTrunc(LOffset, RHSIndexTy); | ||||||||
| 949 | } | ||||||||
| 950 | |||||||||
| 951 | Value *Cmp = Builder.CreateICmp(ICmpInst::getSignedPredicate(Cond), | ||||||||
| 952 | LOffset, ROffset); | ||||||||
| 953 | return replaceInstUsesWith(I, Cmp); | ||||||||
| 954 | } | ||||||||
| 955 | |||||||||
| 956 | // Otherwise, the base pointers are different and the indices are | ||||||||
| 957 | // different. Try convert this to an indexed compare by looking through | ||||||||
| 958 | // PHIs/casts. | ||||||||
| 959 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | ||||||||
| 960 | } | ||||||||
| 961 | |||||||||
| 962 | // If one of the GEPs has all zero indices, recurse. | ||||||||
| 963 | // FIXME: Handle vector of pointers. | ||||||||
| 964 | if (!GEPLHS->getType()->isVectorTy() && GEPLHS->hasAllZeroIndices()) | ||||||||
| 965 | return foldGEPICmp(GEPRHS, GEPLHS->getOperand(0), | ||||||||
| 966 | ICmpInst::getSwappedPredicate(Cond), I); | ||||||||
| 967 | |||||||||
| 968 | // If the other GEP has all zero indices, recurse. | ||||||||
| 969 | // FIXME: Handle vector of pointers. | ||||||||
| 970 | if (!GEPRHS->getType()->isVectorTy() && GEPRHS->hasAllZeroIndices()) | ||||||||
| 971 | return foldGEPICmp(GEPLHS, GEPRHS->getOperand(0), Cond, I); | ||||||||
| 972 | |||||||||
| 973 | bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds(); | ||||||||
| 974 | if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands()) { | ||||||||
| 975 | // If the GEPs only differ by one index, compare it. | ||||||||
| 976 | unsigned NumDifferences = 0; // Keep track of # differences. | ||||||||
| 977 | unsigned DiffOperand = 0; // The operand that differs. | ||||||||
| 978 | for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i) | ||||||||
| 979 | if (GEPLHS->getOperand(i) != GEPRHS->getOperand(i)) { | ||||||||
| 980 | Type *LHSType = GEPLHS->getOperand(i)->getType(); | ||||||||
| 981 | Type *RHSType = GEPRHS->getOperand(i)->getType(); | ||||||||
| 982 | // FIXME: Better support for vector of pointers. | ||||||||
| 983 | if (LHSType->getPrimitiveSizeInBits() != | ||||||||
| 984 | RHSType->getPrimitiveSizeInBits() || | ||||||||
| 985 | (GEPLHS->getType()->isVectorTy() && | ||||||||
| 986 | (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) { | ||||||||
| 987 | // Irreconcilable differences. | ||||||||
| 988 | NumDifferences = 2; | ||||||||
| 989 | break; | ||||||||
| 990 | } | ||||||||
| 991 | |||||||||
| 992 | if (NumDifferences++) break; | ||||||||
| 993 | DiffOperand = i; | ||||||||
| 994 | } | ||||||||
| 995 | |||||||||
| 996 | if (NumDifferences == 0) // SAME GEP? | ||||||||
| 997 | return replaceInstUsesWith(I, // No comparison is needed here. | ||||||||
| 998 | ConstantInt::get(I.getType(), ICmpInst::isTrueWhenEqual(Cond))); | ||||||||
| 999 | |||||||||
| 1000 | else if (NumDifferences == 1 && GEPsInBounds) { | ||||||||
| 1001 | Value *LHSV = GEPLHS->getOperand(DiffOperand); | ||||||||
| 1002 | Value *RHSV = GEPRHS->getOperand(DiffOperand); | ||||||||
| 1003 | // Make sure we do a signed comparison here. | ||||||||
| 1004 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), LHSV, RHSV); | ||||||||
| 1005 | } | ||||||||
| 1006 | } | ||||||||
| 1007 | |||||||||
| 1008 | // Only lower this if the icmp is the only user of the GEP or if we expect | ||||||||
| 1009 | // the result to fold to a constant! | ||||||||
| 1010 | if (GEPsInBounds && (isa<ConstantExpr>(GEPLHS) || GEPLHS->hasOneUse()) && | ||||||||
| 1011 | (isa<ConstantExpr>(GEPRHS) || GEPRHS->hasOneUse())) { | ||||||||
| 1012 | // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) | ||||||||
| 1013 | Value *L = EmitGEPOffset(GEPLHS); | ||||||||
| 1014 | Value *R = EmitGEPOffset(GEPRHS); | ||||||||
| 1015 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R); | ||||||||
| 1016 | } | ||||||||
| 1017 | } | ||||||||
| 1018 | |||||||||
| 1019 | // Try convert this to an indexed compare by looking through PHIs/casts as a | ||||||||
| 1020 | // last resort. | ||||||||
| 1021 | return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); | ||||||||
| 1022 | } | ||||||||
| 1023 | |||||||||
| 1024 | Instruction *InstCombiner::foldAllocaCmp(ICmpInst &ICI, | ||||||||
| 1025 | const AllocaInst *Alloca, | ||||||||
| 1026 | const Value *Other) { | ||||||||
| 1027 | assert(ICI.isEquality() && "Cannot fold non-equality comparison.")((ICI.isEquality() && "Cannot fold non-equality comparison." ) ? static_cast<void> (0) : __assert_fail ("ICI.isEquality() && \"Cannot fold non-equality comparison.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1027, __PRETTY_FUNCTION__)); | ||||||||
| 1028 | |||||||||
| 1029 | // It would be tempting to fold away comparisons between allocas and any | ||||||||
| 1030 | // pointer not based on that alloca (e.g. an argument). However, even | ||||||||
| 1031 | // though such pointers cannot alias, they can still compare equal. | ||||||||
| 1032 | // | ||||||||
| 1033 | // But LLVM doesn't specify where allocas get their memory, so if the alloca | ||||||||
| 1034 | // doesn't escape we can argue that it's impossible to guess its value, and we | ||||||||
| 1035 | // can therefore act as if any such guesses are wrong. | ||||||||
| 1036 | // | ||||||||
| 1037 | // The code below checks that the alloca doesn't escape, and that it's only | ||||||||
| 1038 | // used in a comparison once (the current instruction). The | ||||||||
| 1039 | // single-comparison-use condition ensures that we're trivially folding all | ||||||||
| 1040 | // comparisons against the alloca consistently, and avoids the risk of | ||||||||
| 1041 | // erroneously folding a comparison of the pointer with itself. | ||||||||
| 1042 | |||||||||
| 1043 | unsigned MaxIter = 32; // Break cycles and bound to constant-time. | ||||||||
| 1044 | |||||||||
| 1045 | SmallVector<const Use *, 32> Worklist; | ||||||||
| 1046 | for (const Use &U : Alloca->uses()) { | ||||||||
| 1047 | if (Worklist.size() >= MaxIter) | ||||||||
| 1048 | return nullptr; | ||||||||
| 1049 | Worklist.push_back(&U); | ||||||||
| 1050 | } | ||||||||
| 1051 | |||||||||
| 1052 | unsigned NumCmps = 0; | ||||||||
| 1053 | while (!Worklist.empty()) { | ||||||||
| 1054 | assert(Worklist.size() <= MaxIter)((Worklist.size() <= MaxIter) ? static_cast<void> (0 ) : __assert_fail ("Worklist.size() <= MaxIter", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1054, __PRETTY_FUNCTION__)); | ||||||||
| 1055 | const Use *U = Worklist.pop_back_val(); | ||||||||
| 1056 | const Value *V = U->getUser(); | ||||||||
| 1057 | --MaxIter; | ||||||||
| 1058 | |||||||||
| 1059 | if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V) || isa<PHINode>(V) || | ||||||||
| 1060 | isa<SelectInst>(V)) { | ||||||||
| 1061 | // Track the uses. | ||||||||
| 1062 | } else if (isa<LoadInst>(V)) { | ||||||||
| 1063 | // Loading from the pointer doesn't escape it. | ||||||||
| 1064 | continue; | ||||||||
| 1065 | } else if (const auto *SI = dyn_cast<StoreInst>(V)) { | ||||||||
| 1066 | // Storing *to* the pointer is fine, but storing the pointer escapes it. | ||||||||
| 1067 | if (SI->getValueOperand() == U->get()) | ||||||||
| 1068 | return nullptr; | ||||||||
| 1069 | continue; | ||||||||
| 1070 | } else if (isa<ICmpInst>(V)) { | ||||||||
| 1071 | if (NumCmps++) | ||||||||
| 1072 | return nullptr; // Found more than one cmp. | ||||||||
| 1073 | continue; | ||||||||
| 1074 | } else if (const auto *Intrin = dyn_cast<IntrinsicInst>(V)) { | ||||||||
| 1075 | switch (Intrin->getIntrinsicID()) { | ||||||||
| 1076 | // These intrinsics don't escape or compare the pointer. Memset is safe | ||||||||
| 1077 | // because we don't allow ptrtoint. Memcpy and memmove are safe because | ||||||||
| 1078 | // we don't allow stores, so src cannot point to V. | ||||||||
| 1079 | case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: | ||||||||
| 1080 | case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: | ||||||||
| 1081 | continue; | ||||||||
| 1082 | default: | ||||||||
| 1083 | return nullptr; | ||||||||
| 1084 | } | ||||||||
| 1085 | } else { | ||||||||
| 1086 | return nullptr; | ||||||||
| 1087 | } | ||||||||
| 1088 | for (const Use &U : V->uses()) { | ||||||||
| 1089 | if (Worklist.size() >= MaxIter) | ||||||||
| 1090 | return nullptr; | ||||||||
| 1091 | Worklist.push_back(&U); | ||||||||
| 1092 | } | ||||||||
| 1093 | } | ||||||||
| 1094 | |||||||||
| 1095 | Type *CmpTy = CmpInst::makeCmpResultType(Other->getType()); | ||||||||
| 1096 | return replaceInstUsesWith( | ||||||||
| 1097 | ICI, | ||||||||
| 1098 | ConstantInt::get(CmpTy, !CmpInst::isTrueWhenEqual(ICI.getPredicate()))); | ||||||||
| 1099 | } | ||||||||
| 1100 | |||||||||
| 1101 | /// Fold "icmp pred (X+C), X". | ||||||||
| 1102 | Instruction *InstCombiner::foldICmpAddOpConst(Value *X, const APInt &C, | ||||||||
| 1103 | ICmpInst::Predicate Pred) { | ||||||||
| 1104 | // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0, | ||||||||
| 1105 | // so the values can never be equal. Similarly for all other "or equals" | ||||||||
| 1106 | // operators. | ||||||||
| 1107 | assert(!!C && "C should not be zero!")((!!C && "C should not be zero!") ? static_cast<void > (0) : __assert_fail ("!!C && \"C should not be zero!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1107, __PRETTY_FUNCTION__)); | ||||||||
| 1108 | |||||||||
| 1109 | // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255 | ||||||||
| 1110 | // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253 | ||||||||
| 1111 | // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0 | ||||||||
| 1112 | if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) { | ||||||||
| 1113 | Constant *R = ConstantInt::get(X->getType(), | ||||||||
| 1114 | APInt::getMaxValue(C.getBitWidth()) - C); | ||||||||
| 1115 | return new ICmpInst(ICmpInst::ICMP_UGT, X, R); | ||||||||
| 1116 | } | ||||||||
| 1117 | |||||||||
| 1118 | // (X+1) >u X --> X <u (0-1) --> X != 255 | ||||||||
| 1119 | // (X+2) >u X --> X <u (0-2) --> X <u 254 | ||||||||
| 1120 | // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0 | ||||||||
| 1121 | if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) | ||||||||
| 1122 | return new ICmpInst(ICmpInst::ICMP_ULT, X, | ||||||||
| 1123 | ConstantInt::get(X->getType(), -C)); | ||||||||
| 1124 | |||||||||
| 1125 | APInt SMax = APInt::getSignedMaxValue(C.getBitWidth()); | ||||||||
| 1126 | |||||||||
| 1127 | // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127 | ||||||||
| 1128 | // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125 | ||||||||
| 1129 | // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0 | ||||||||
| 1130 | // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1 | ||||||||
| 1131 | // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126 | ||||||||
| 1132 | // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127 | ||||||||
| 1133 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | ||||||||
| 1134 | return new ICmpInst(ICmpInst::ICMP_SGT, X, | ||||||||
| 1135 | ConstantInt::get(X->getType(), SMax - C)); | ||||||||
| 1136 | |||||||||
| 1137 | // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127 | ||||||||
| 1138 | // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126 | ||||||||
| 1139 | // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1 | ||||||||
| 1140 | // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2 | ||||||||
| 1141 | // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126 | ||||||||
| 1142 | // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128 | ||||||||
| 1143 | |||||||||
| 1144 | assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) ? static_cast<void> (0) : __assert_fail ("Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1144, __PRETTY_FUNCTION__)); | ||||||||
| 1145 | return new ICmpInst(ICmpInst::ICMP_SLT, X, | ||||||||
| 1146 | ConstantInt::get(X->getType(), SMax - (C - 1))); | ||||||||
| 1147 | } | ||||||||
| 1148 | |||||||||
| 1149 | /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" -> | ||||||||
| 1150 | /// (icmp eq/ne A, Log2(AP2/AP1)) -> | ||||||||
| 1151 | /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)). | ||||||||
| 1152 | Instruction *InstCombiner::foldICmpShrConstConst(ICmpInst &I, Value *A, | ||||||||
| 1153 | const APInt &AP1, | ||||||||
| 1154 | const APInt &AP2) { | ||||||||
| 1155 | assert(I.isEquality() && "Cannot fold icmp gt/lt")((I.isEquality() && "Cannot fold icmp gt/lt") ? static_cast <void> (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1155, __PRETTY_FUNCTION__)); | ||||||||
| 1156 | |||||||||
| 1157 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | ||||||||
| 1158 | if (I.getPredicate() == I.ICMP_NE) | ||||||||
| 1159 | Pred = CmpInst::getInversePredicate(Pred); | ||||||||
| 1160 | return new ICmpInst(Pred, LHS, RHS); | ||||||||
| 1161 | }; | ||||||||
| 1162 | |||||||||
| 1163 | // Don't bother doing any work for cases which InstSimplify handles. | ||||||||
| 1164 | if (AP2.isNullValue()) | ||||||||
| 1165 | return nullptr; | ||||||||
| 1166 | |||||||||
| 1167 | bool IsAShr = isa<AShrOperator>(I.getOperand(0)); | ||||||||
| 1168 | if (IsAShr) { | ||||||||
| 1169 | if (AP2.isAllOnesValue()) | ||||||||
| 1170 | return nullptr; | ||||||||
| 1171 | if (AP2.isNegative() != AP1.isNegative()) | ||||||||
| 1172 | return nullptr; | ||||||||
| 1173 | if (AP2.sgt(AP1)) | ||||||||
| 1174 | return nullptr; | ||||||||
| 1175 | } | ||||||||
| 1176 | |||||||||
| 1177 | if (!AP1) | ||||||||
| 1178 | // 'A' must be large enough to shift out the highest set bit. | ||||||||
| 1179 | return getICmp(I.ICMP_UGT, A, | ||||||||
| 1180 | ConstantInt::get(A->getType(), AP2.logBase2())); | ||||||||
| 1181 | |||||||||
| 1182 | if (AP1 == AP2) | ||||||||
| 1183 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | ||||||||
| 1184 | |||||||||
| 1185 | int Shift; | ||||||||
| 1186 | if (IsAShr && AP1.isNegative()) | ||||||||
| 1187 | Shift = AP1.countLeadingOnes() - AP2.countLeadingOnes(); | ||||||||
| 1188 | else | ||||||||
| 1189 | Shift = AP1.countLeadingZeros() - AP2.countLeadingZeros(); | ||||||||
| 1190 | |||||||||
| 1191 | if (Shift > 0) { | ||||||||
| 1192 | if (IsAShr && AP1 == AP2.ashr(Shift)) { | ||||||||
| 1193 | // There are multiple solutions if we are comparing against -1 and the LHS | ||||||||
| 1194 | // of the ashr is not a power of two. | ||||||||
| 1195 | if (AP1.isAllOnesValue() && !AP2.isPowerOf2()) | ||||||||
| 1196 | return getICmp(I.ICMP_UGE, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
| 1197 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
| 1198 | } else if (AP1 == AP2.lshr(Shift)) { | ||||||||
| 1199 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
| 1200 | } | ||||||||
| 1201 | } | ||||||||
| 1202 | |||||||||
| 1203 | // Shifting const2 will never be equal to const1. | ||||||||
| 1204 | // FIXME: This should always be handled by InstSimplify? | ||||||||
| 1205 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | ||||||||
| 1206 | return replaceInstUsesWith(I, TorF); | ||||||||
| 1207 | } | ||||||||
| 1208 | |||||||||
| 1209 | /// Handle "(icmp eq/ne (shl AP2, A), AP1)" -> | ||||||||
| 1210 | /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)). | ||||||||
| 1211 | Instruction *InstCombiner::foldICmpShlConstConst(ICmpInst &I, Value *A, | ||||||||
| 1212 | const APInt &AP1, | ||||||||
| 1213 | const APInt &AP2) { | ||||||||
| 1214 | assert(I.isEquality() && "Cannot fold icmp gt/lt")((I.isEquality() && "Cannot fold icmp gt/lt") ? static_cast <void> (0) : __assert_fail ("I.isEquality() && \"Cannot fold icmp gt/lt\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1214, __PRETTY_FUNCTION__)); | ||||||||
| 1215 | |||||||||
| 1216 | auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) { | ||||||||
| 1217 | if (I.getPredicate() == I.ICMP_NE) | ||||||||
| 1218 | Pred = CmpInst::getInversePredicate(Pred); | ||||||||
| 1219 | return new ICmpInst(Pred, LHS, RHS); | ||||||||
| 1220 | }; | ||||||||
| 1221 | |||||||||
| 1222 | // Don't bother doing any work for cases which InstSimplify handles. | ||||||||
| 1223 | if (AP2.isNullValue()) | ||||||||
| 1224 | return nullptr; | ||||||||
| 1225 | |||||||||
| 1226 | unsigned AP2TrailingZeros = AP2.countTrailingZeros(); | ||||||||
| 1227 | |||||||||
| 1228 | if (!AP1 && AP2TrailingZeros != 0) | ||||||||
| 1229 | return getICmp( | ||||||||
| 1230 | I.ICMP_UGE, A, | ||||||||
| 1231 | ConstantInt::get(A->getType(), AP2.getBitWidth() - AP2TrailingZeros)); | ||||||||
| 1232 | |||||||||
| 1233 | if (AP1 == AP2) | ||||||||
| 1234 | return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(A->getType())); | ||||||||
| 1235 | |||||||||
| 1236 | // Get the distance between the lowest bits that are set. | ||||||||
| 1237 | int Shift = AP1.countTrailingZeros() - AP2TrailingZeros; | ||||||||
| 1238 | |||||||||
| 1239 | if (Shift > 0 && AP2.shl(Shift) == AP1) | ||||||||
| 1240 | return getICmp(I.ICMP_EQ, A, ConstantInt::get(A->getType(), Shift)); | ||||||||
| 1241 | |||||||||
| 1242 | // Shifting const2 will never be equal to const1. | ||||||||
| 1243 | // FIXME: This should always be handled by InstSimplify? | ||||||||
| 1244 | auto *TorF = ConstantInt::get(I.getType(), I.getPredicate() == I.ICMP_NE); | ||||||||
| 1245 | return replaceInstUsesWith(I, TorF); | ||||||||
| 1246 | } | ||||||||
| 1247 | |||||||||
| 1248 | /// The caller has matched a pattern of the form: | ||||||||
| 1249 | /// I = icmp ugt (add (add A, B), CI2), CI1 | ||||||||
| 1250 | /// If this is of the form: | ||||||||
| 1251 | /// sum = a + b | ||||||||
| 1252 | /// if (sum+128 >u 255) | ||||||||
| 1253 | /// Then replace it with llvm.sadd.with.overflow.i8. | ||||||||
| 1254 | /// | ||||||||
| 1255 | static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B, | ||||||||
| 1256 | ConstantInt *CI2, ConstantInt *CI1, | ||||||||
| 1257 | InstCombiner &IC) { | ||||||||
| 1258 | // The transformation we're trying to do here is to transform this into an | ||||||||
| 1259 | // llvm.sadd.with.overflow. To do this, we have to replace the original add | ||||||||
| 1260 | // with a narrower add, and discard the add-with-constant that is part of the | ||||||||
| 1261 | // range check (if we can't eliminate it, this isn't profitable). | ||||||||
| 1262 | |||||||||
| 1263 | // In order to eliminate the add-with-constant, the compare can be its only | ||||||||
| 1264 | // use. | ||||||||
| 1265 | Instruction *AddWithCst = cast<Instruction>(I.getOperand(0)); | ||||||||
| 1266 | if (!AddWithCst->hasOneUse()) | ||||||||
| 1267 | return nullptr; | ||||||||
| 1268 | |||||||||
| 1269 | // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow. | ||||||||
| 1270 | if (!CI2->getValue().isPowerOf2()) | ||||||||
| 1271 | return nullptr; | ||||||||
| 1272 | unsigned NewWidth = CI2->getValue().countTrailingZeros(); | ||||||||
| 1273 | if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31) | ||||||||
| 1274 | return nullptr; | ||||||||
| 1275 | |||||||||
| 1276 | // The width of the new add formed is 1 more than the bias. | ||||||||
| 1277 | ++NewWidth; | ||||||||
| 1278 | |||||||||
| 1279 | // Check to see that CI1 is an all-ones value with NewWidth bits. | ||||||||
| 1280 | if (CI1->getBitWidth() == NewWidth || | ||||||||
| 1281 | CI1->getValue() != APInt::getLowBitsSet(CI1->getBitWidth(), NewWidth)) | ||||||||
| 1282 | return nullptr; | ||||||||
| 1283 | |||||||||
| 1284 | // This is only really a signed overflow check if the inputs have been | ||||||||
| 1285 | // sign-extended; check for that condition. For example, if CI2 is 2^31 and | ||||||||
| 1286 | // the operands of the add are 64 bits wide, we need at least 33 sign bits. | ||||||||
| 1287 | unsigned NeededSignBits = CI1->getBitWidth() - NewWidth + 1; | ||||||||
| 1288 | if (IC.ComputeNumSignBits(A, 0, &I) < NeededSignBits || | ||||||||
| 1289 | IC.ComputeNumSignBits(B, 0, &I) < NeededSignBits) | ||||||||
| 1290 | return nullptr; | ||||||||
| 1291 | |||||||||
| 1292 | // In order to replace the original add with a narrower | ||||||||
| 1293 | // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant | ||||||||
| 1294 | // and truncates that discard the high bits of the add. Verify that this is | ||||||||
| 1295 | // the case. | ||||||||
| 1296 | Instruction *OrigAdd = cast<Instruction>(AddWithCst->getOperand(0)); | ||||||||
| 1297 | for (User *U : OrigAdd->users()) { | ||||||||
| 1298 | if (U == AddWithCst) | ||||||||
| 1299 | continue; | ||||||||
| 1300 | |||||||||
| 1301 | // Only accept truncates for now. We would really like a nice recursive | ||||||||
| 1302 | // predicate like SimplifyDemandedBits, but which goes downwards the use-def | ||||||||
| 1303 | // chain to see which bits of a value are actually demanded. If the | ||||||||
| 1304 | // original add had another add which was then immediately truncated, we | ||||||||
| 1305 | // could still do the transformation. | ||||||||
| 1306 | TruncInst *TI = dyn_cast<TruncInst>(U); | ||||||||
| 1307 | if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth) | ||||||||
| 1308 | return nullptr; | ||||||||
| 1309 | } | ||||||||
| 1310 | |||||||||
| 1311 | // If the pattern matches, truncate the inputs to the narrower type and | ||||||||
| 1312 | // use the sadd_with_overflow intrinsic to efficiently compute both the | ||||||||
| 1313 | // result and the overflow bit. | ||||||||
| 1314 | Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth); | ||||||||
| 1315 | Function *F = Intrinsic::getDeclaration( | ||||||||
| 1316 | I.getModule(), Intrinsic::sadd_with_overflow, NewType); | ||||||||
| 1317 | |||||||||
| 1318 | InstCombiner::BuilderTy &Builder = IC.Builder; | ||||||||
| 1319 | |||||||||
| 1320 | // Put the new code above the original add, in case there are any uses of the | ||||||||
| 1321 | // add between the add and the compare. | ||||||||
| 1322 | Builder.SetInsertPoint(OrigAdd); | ||||||||
| 1323 | |||||||||
| 1324 | Value *TruncA = Builder.CreateTrunc(A, NewType, A->getName() + ".trunc"); | ||||||||
| 1325 | Value *TruncB = Builder.CreateTrunc(B, NewType, B->getName() + ".trunc"); | ||||||||
| 1326 | CallInst *Call = Builder.CreateCall(F, {TruncA, TruncB}, "sadd"); | ||||||||
| 1327 | Value *Add = Builder.CreateExtractValue(Call, 0, "sadd.result"); | ||||||||
| 1328 | Value *ZExt = Builder.CreateZExt(Add, OrigAdd->getType()); | ||||||||
| 1329 | |||||||||
| 1330 | // The inner add was the result of the narrow add, zero extended to the | ||||||||
| 1331 | // wider type. Replace it with the result computed by the intrinsic. | ||||||||
| 1332 | IC.replaceInstUsesWith(*OrigAdd, ZExt); | ||||||||
| 1333 | |||||||||
| 1334 | // The original icmp gets replaced with the overflow value. | ||||||||
| 1335 | return ExtractValueInst::Create(Call, 1, "sadd.overflow"); | ||||||||
| 1336 | } | ||||||||
| 1337 | |||||||||
| 1338 | /// If we have: | ||||||||
| 1339 | /// icmp eq/ne (urem/srem %x, %y), 0 | ||||||||
| 1340 | /// iff %y is a power-of-two, we can replace this with a bit test: | ||||||||
| 1341 | /// icmp eq/ne (and %x, (add %y, -1)), 0 | ||||||||
| 1342 | Instruction *InstCombiner::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) { | ||||||||
| 1343 | // This fold is only valid for equality predicates. | ||||||||
| 1344 | if (!I.isEquality()) | ||||||||
| 1345 | return nullptr; | ||||||||
| 1346 | ICmpInst::Predicate Pred; | ||||||||
| 1347 | Value *X, *Y, *Zero; | ||||||||
| 1348 | if (!match(&I, m_ICmp(Pred, m_OneUse(m_IRem(m_Value(X), m_Value(Y))), | ||||||||
| 1349 | m_CombineAnd(m_Zero(), m_Value(Zero))))) | ||||||||
| 1350 | return nullptr; | ||||||||
| 1351 | if (!isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, 0, &I)) | ||||||||
| 1352 | return nullptr; | ||||||||
| 1353 | // This may increase instruction count, we don't enforce that Y is a constant. | ||||||||
| 1354 | Value *Mask = Builder.CreateAdd(Y, Constant::getAllOnesValue(Y->getType())); | ||||||||
| 1355 | Value *Masked = Builder.CreateAnd(X, Mask); | ||||||||
| 1356 | return ICmpInst::Create(Instruction::ICmp, Pred, Masked, Zero); | ||||||||
| 1357 | } | ||||||||
| 1358 | |||||||||
| 1359 | /// Fold equality-comparison between zero and any (maybe truncated) right-shift | ||||||||
| 1360 | /// by one-less-than-bitwidth into a sign test on the original value. | ||||||||
| 1361 | Instruction *InstCombiner::foldSignBitTest(ICmpInst &I) { | ||||||||
| 1362 | Instruction *Val; | ||||||||
| 1363 | ICmpInst::Predicate Pred; | ||||||||
| 1364 | if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero()))) | ||||||||
| 1365 | return nullptr; | ||||||||
| 1366 | |||||||||
| 1367 | Value *X; | ||||||||
| 1368 | Type *XTy; | ||||||||
| 1369 | |||||||||
| 1370 | Constant *C; | ||||||||
| 1371 | if (match(Val, m_TruncOrSelf(m_Shr(m_Value(X), m_Constant(C))))) { | ||||||||
| 1372 | XTy = X->getType(); | ||||||||
| 1373 | unsigned XBitWidth = XTy->getScalarSizeInBits(); | ||||||||
| 1374 | if (!match(C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ, | ||||||||
| 1375 | APInt(XBitWidth, XBitWidth - 1)))) | ||||||||
| 1376 | return nullptr; | ||||||||
| 1377 | } else if (isa<BinaryOperator>(Val) && | ||||||||
| 1378 | (X = reassociateShiftAmtsOfTwoSameDirectionShifts( | ||||||||
| 1379 | cast<BinaryOperator>(Val), SQ.getWithInstruction(Val), | ||||||||
| 1380 | /*AnalyzeForSignBitExtraction=*/true))) { | ||||||||
| 1381 | XTy = X->getType(); | ||||||||
| 1382 | } else | ||||||||
| 1383 | return nullptr; | ||||||||
| 1384 | |||||||||
| 1385 | return ICmpInst::Create(Instruction::ICmp, | ||||||||
| 1386 | Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE | ||||||||
| 1387 | : ICmpInst::ICMP_SLT, | ||||||||
| 1388 | X, ConstantInt::getNullValue(XTy)); | ||||||||
| 1389 | } | ||||||||
| 1390 | |||||||||
| 1391 | // Handle icmp pred X, 0 | ||||||||
| 1392 | Instruction *InstCombiner::foldICmpWithZero(ICmpInst &Cmp) { | ||||||||
| 1393 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1394 | if (!match(Cmp.getOperand(1), m_Zero())) | ||||||||
| 1395 | return nullptr; | ||||||||
| 1396 | |||||||||
| 1397 | // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0) | ||||||||
| 1398 | if (Pred == ICmpInst::ICMP_SGT) { | ||||||||
| 1399 | Value *A, *B; | ||||||||
| 1400 | SelectPatternResult SPR = matchSelectPattern(Cmp.getOperand(0), A, B); | ||||||||
| 1401 | if (SPR.Flavor == SPF_SMIN) { | ||||||||
| 1402 | if (isKnownPositive(A, DL, 0, &AC, &Cmp, &DT)) | ||||||||
| 1403 | return new ICmpInst(Pred, B, Cmp.getOperand(1)); | ||||||||
| 1404 | if (isKnownPositive(B, DL, 0, &AC, &Cmp, &DT)) | ||||||||
| 1405 | return new ICmpInst(Pred, A, Cmp.getOperand(1)); | ||||||||
| 1406 | } | ||||||||
| 1407 | } | ||||||||
| 1408 | |||||||||
| 1409 | if (Instruction *New = foldIRemByPowerOfTwoToBitTest(Cmp)) | ||||||||
| 1410 | return New; | ||||||||
| 1411 | |||||||||
| 1412 | // Given: | ||||||||
| 1413 | // icmp eq/ne (urem %x, %y), 0 | ||||||||
| 1414 | // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem': | ||||||||
| 1415 | // icmp eq/ne %x, 0 | ||||||||
| 1416 | Value *X, *Y; | ||||||||
| 1417 | if (match(Cmp.getOperand(0), m_URem(m_Value(X), m_Value(Y))) && | ||||||||
| 1418 | ICmpInst::isEquality(Pred)) { | ||||||||
| 1419 | KnownBits XKnown = computeKnownBits(X, 0, &Cmp); | ||||||||
| 1420 | KnownBits YKnown = computeKnownBits(Y, 0, &Cmp); | ||||||||
| 1421 | if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2) | ||||||||
| 1422 | return new ICmpInst(Pred, X, Cmp.getOperand(1)); | ||||||||
| 1423 | } | ||||||||
| 1424 | |||||||||
| 1425 | return nullptr; | ||||||||
| 1426 | } | ||||||||
| 1427 | |||||||||
| 1428 | /// Fold icmp Pred X, C. | ||||||||
| 1429 | /// TODO: This code structure does not make sense. The saturating add fold | ||||||||
| 1430 | /// should be moved to some other helper and extended as noted below (it is also | ||||||||
| 1431 | /// possible that code has been made unnecessary - do we canonicalize IR to | ||||||||
| 1432 | /// overflow/saturating intrinsics or not?). | ||||||||
| 1433 | Instruction *InstCombiner::foldICmpWithConstant(ICmpInst &Cmp) { | ||||||||
| 1434 | // Match the following pattern, which is a common idiom when writing | ||||||||
| 1435 | // overflow-safe integer arithmetic functions. The source performs an addition | ||||||||
| 1436 | // in wider type and explicitly checks for overflow using comparisons against | ||||||||
| 1437 | // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic. | ||||||||
| 1438 | // | ||||||||
| 1439 | // TODO: This could probably be generalized to handle other overflow-safe | ||||||||
| 1440 | // operations if we worked out the formulas to compute the appropriate magic | ||||||||
| 1441 | // constants. | ||||||||
| 1442 | // | ||||||||
| 1443 | // sum = a + b | ||||||||
| 1444 | // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8 | ||||||||
| 1445 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1446 | Value *Op0 = Cmp.getOperand(0), *Op1 = Cmp.getOperand(1); | ||||||||
| 1447 | Value *A, *B; | ||||||||
| 1448 | ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI | ||||||||
| 1449 | if (Pred == ICmpInst::ICMP_UGT && match(Op1, m_ConstantInt(CI)) && | ||||||||
| 1450 | match(Op0, m_Add(m_Add(m_Value(A), m_Value(B)), m_ConstantInt(CI2)))) | ||||||||
| 1451 | if (Instruction *Res = processUGT_ADDCST_ADD(Cmp, A, B, CI2, CI, *this)) | ||||||||
| 1452 | return Res; | ||||||||
| 1453 | |||||||||
| 1454 | return nullptr; | ||||||||
| 1455 | } | ||||||||
| 1456 | |||||||||
| 1457 | /// Canonicalize icmp instructions based on dominating conditions. | ||||||||
| 1458 | Instruction *InstCombiner::foldICmpWithDominatingICmp(ICmpInst &Cmp) { | ||||||||
| 1459 | // This is a cheap/incomplete check for dominance - just match a single | ||||||||
| 1460 | // predecessor with a conditional branch. | ||||||||
| 1461 | BasicBlock *CmpBB = Cmp.getParent(); | ||||||||
| 1462 | BasicBlock *DomBB = CmpBB->getSinglePredecessor(); | ||||||||
| 1463 | if (!DomBB) | ||||||||
| 1464 | return nullptr; | ||||||||
| 1465 | |||||||||
| 1466 | Value *DomCond; | ||||||||
| 1467 | BasicBlock *TrueBB, *FalseBB; | ||||||||
| 1468 | if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB))) | ||||||||
| 1469 | return nullptr; | ||||||||
| 1470 | |||||||||
| 1471 | assert((TrueBB == CmpBB || FalseBB == CmpBB) &&(((TrueBB == CmpBB || FalseBB == CmpBB) && "Predecessor block does not point to successor?" ) ? static_cast<void> (0) : __assert_fail ("(TrueBB == CmpBB || FalseBB == CmpBB) && \"Predecessor block does not point to successor?\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1472, __PRETTY_FUNCTION__)) | ||||||||
| 1472 | "Predecessor block does not point to successor?")(((TrueBB == CmpBB || FalseBB == CmpBB) && "Predecessor block does not point to successor?" ) ? static_cast<void> (0) : __assert_fail ("(TrueBB == CmpBB || FalseBB == CmpBB) && \"Predecessor block does not point to successor?\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1472, __PRETTY_FUNCTION__)); | ||||||||
| 1473 | |||||||||
| 1474 | // The branch should get simplified. Don't bother simplifying this condition. | ||||||||
| 1475 | if (TrueBB == FalseBB) | ||||||||
| 1476 | return nullptr; | ||||||||
| 1477 | |||||||||
| 1478 | // Try to simplify this compare to T/F based on the dominating condition. | ||||||||
| 1479 | Optional<bool> Imp = isImpliedCondition(DomCond, &Cmp, DL, TrueBB == CmpBB); | ||||||||
| 1480 | if (Imp) | ||||||||
| 1481 | return replaceInstUsesWith(Cmp, ConstantInt::get(Cmp.getType(), *Imp)); | ||||||||
| 1482 | |||||||||
| 1483 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1484 | Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1); | ||||||||
| 1485 | ICmpInst::Predicate DomPred; | ||||||||
| 1486 | const APInt *C, *DomC; | ||||||||
| 1487 | if (match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))) && | ||||||||
| 1488 | match(Y, m_APInt(C))) { | ||||||||
| 1489 | // We have 2 compares of a variable with constants. Calculate the constant | ||||||||
| 1490 | // ranges of those compares to see if we can transform the 2nd compare: | ||||||||
| 1491 | // DomBB: | ||||||||
| 1492 | // DomCond = icmp DomPred X, DomC | ||||||||
| 1493 | // br DomCond, CmpBB, FalseBB | ||||||||
| 1494 | // CmpBB: | ||||||||
| 1495 | // Cmp = icmp Pred X, C | ||||||||
| 1496 | ConstantRange CR = ConstantRange::makeAllowedICmpRegion(Pred, *C); | ||||||||
| 1497 | ConstantRange DominatingCR = | ||||||||
| 1498 | (CmpBB == TrueBB) ? ConstantRange::makeExactICmpRegion(DomPred, *DomC) | ||||||||
| 1499 | : ConstantRange::makeExactICmpRegion( | ||||||||
| 1500 | CmpInst::getInversePredicate(DomPred), *DomC); | ||||||||
| 1501 | ConstantRange Intersection = DominatingCR.intersectWith(CR); | ||||||||
| 1502 | ConstantRange Difference = DominatingCR.difference(CR); | ||||||||
| 1503 | if (Intersection.isEmptySet()) | ||||||||
| 1504 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
| 1505 | if (Difference.isEmptySet()) | ||||||||
| 1506 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
| 1507 | |||||||||
| 1508 | // Canonicalizing a sign bit comparison that gets used in a branch, | ||||||||
| 1509 | // pessimizes codegen by generating branch on zero instruction instead | ||||||||
| 1510 | // of a test and branch. So we avoid canonicalizing in such situations | ||||||||
| 1511 | // because test and branch instruction has better branch displacement | ||||||||
| 1512 | // than compare and branch instruction. | ||||||||
| 1513 | bool UnusedBit; | ||||||||
| 1514 | bool IsSignBit = isSignBitCheck(Pred, *C, UnusedBit); | ||||||||
| 1515 | if (Cmp.isEquality() || (IsSignBit && hasBranchUse(Cmp))) | ||||||||
| 1516 | return nullptr; | ||||||||
| 1517 | |||||||||
| 1518 | if (const APInt *EqC = Intersection.getSingleElement()) | ||||||||
| 1519 | return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC)); | ||||||||
| 1520 | if (const APInt *NeC = Difference.getSingleElement()) | ||||||||
| 1521 | return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC)); | ||||||||
| 1522 | } | ||||||||
| 1523 | |||||||||
| 1524 | return nullptr; | ||||||||
| 1525 | } | ||||||||
| 1526 | |||||||||
| 1527 | /// Fold icmp (trunc X, Y), C. | ||||||||
| 1528 | Instruction *InstCombiner::foldICmpTruncConstant(ICmpInst &Cmp, | ||||||||
| 1529 | TruncInst *Trunc, | ||||||||
| 1530 | const APInt &C) { | ||||||||
| 1531 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1532 | Value *X = Trunc->getOperand(0); | ||||||||
| 1533 | if (C.isOneValue() && C.getBitWidth() > 1) { | ||||||||
| 1534 | // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1 | ||||||||
| 1535 | Value *V = nullptr; | ||||||||
| 1536 | if (Pred == ICmpInst::ICMP_SLT && match(X, m_Signum(m_Value(V)))) | ||||||||
| 1537 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | ||||||||
| 1538 | ConstantInt::get(V->getType(), 1)); | ||||||||
| 1539 | } | ||||||||
| 1540 | |||||||||
| 1541 | if (Cmp.isEquality() && Trunc->hasOneUse()) { | ||||||||
| 1542 | // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all | ||||||||
| 1543 | // of the high bits truncated out of x are known. | ||||||||
| 1544 | unsigned DstBits = Trunc->getType()->getScalarSizeInBits(), | ||||||||
| 1545 | SrcBits = X->getType()->getScalarSizeInBits(); | ||||||||
| 1546 | KnownBits Known = computeKnownBits(X, 0, &Cmp); | ||||||||
| 1547 | |||||||||
| 1548 | // If all the high bits are known, we can do this xform. | ||||||||
| 1549 | if ((Known.Zero | Known.One).countLeadingOnes() >= SrcBits - DstBits) { | ||||||||
| 1550 | // Pull in the high bits from known-ones set. | ||||||||
| 1551 | APInt NewRHS = C.zext(SrcBits); | ||||||||
| 1552 | NewRHS |= Known.One & APInt::getHighBitsSet(SrcBits, SrcBits - DstBits); | ||||||||
| 1553 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), NewRHS)); | ||||||||
| 1554 | } | ||||||||
| 1555 | } | ||||||||
| 1556 | |||||||||
| 1557 | return nullptr; | ||||||||
| 1558 | } | ||||||||
| 1559 | |||||||||
| 1560 | /// Fold icmp (xor X, Y), C. | ||||||||
| 1561 | Instruction *InstCombiner::foldICmpXorConstant(ICmpInst &Cmp, | ||||||||
| 1562 | BinaryOperator *Xor, | ||||||||
| 1563 | const APInt &C) { | ||||||||
| 1564 | Value *X = Xor->getOperand(0); | ||||||||
| 1565 | Value *Y = Xor->getOperand(1); | ||||||||
| 1566 | const APInt *XorC; | ||||||||
| 1567 | if (!match(Y, m_APInt(XorC))) | ||||||||
| 1568 | return nullptr; | ||||||||
| 1569 | |||||||||
| 1570 | // If this is a comparison that tests the signbit (X < 0) or (x > -1), | ||||||||
| 1571 | // fold the xor. | ||||||||
| 1572 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1573 | bool TrueIfSigned = false; | ||||||||
| 1574 | if (isSignBitCheck(Cmp.getPredicate(), C, TrueIfSigned)) { | ||||||||
| 1575 | |||||||||
| 1576 | // If the sign bit of the XorCst is not set, there is no change to | ||||||||
| 1577 | // the operation, just stop using the Xor. | ||||||||
| 1578 | if (!XorC->isNegative()) | ||||||||
| 1579 | return replaceOperand(Cmp, 0, X); | ||||||||
| 1580 | |||||||||
| 1581 | // Emit the opposite comparison. | ||||||||
| 1582 | if (TrueIfSigned) | ||||||||
| 1583 | return new ICmpInst(ICmpInst::ICMP_SGT, X, | ||||||||
| 1584 | ConstantInt::getAllOnesValue(X->getType())); | ||||||||
| 1585 | else | ||||||||
| 1586 | return new ICmpInst(ICmpInst::ICMP_SLT, X, | ||||||||
| 1587 | ConstantInt::getNullValue(X->getType())); | ||||||||
| 1588 | } | ||||||||
| 1589 | |||||||||
| 1590 | if (Xor->hasOneUse()) { | ||||||||
| 1591 | // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask)) | ||||||||
| 1592 | if (!Cmp.isEquality() && XorC->isSignMask()) { | ||||||||
| 1593 | Pred = Cmp.isSigned() ? Cmp.getUnsignedPredicate() | ||||||||
| 1594 | : Cmp.getSignedPredicate(); | ||||||||
| 1595 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); | ||||||||
| 1596 | } | ||||||||
| 1597 | |||||||||
| 1598 | // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask)) | ||||||||
| 1599 | if (!Cmp.isEquality() && XorC->isMaxSignedValue()) { | ||||||||
| 1600 | Pred = Cmp.isSigned() ? Cmp.getUnsignedPredicate() | ||||||||
| 1601 | : Cmp.getSignedPredicate(); | ||||||||
| 1602 | Pred = Cmp.getSwappedPredicate(Pred); | ||||||||
| 1603 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), C ^ *XorC)); | ||||||||
| 1604 | } | ||||||||
| 1605 | } | ||||||||
| 1606 | |||||||||
| 1607 | // Mask constant magic can eliminate an 'xor' with unsigned compares. | ||||||||
| 1608 | if (Pred == ICmpInst::ICMP_UGT) { | ||||||||
| 1609 | // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2) | ||||||||
| 1610 | if (*XorC == ~C && (C + 1).isPowerOf2()) | ||||||||
| 1611 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | ||||||||
| 1612 | // (xor X, C) >u C --> X >u C (when C+1 is a power of 2) | ||||||||
| 1613 | if (*XorC == C && (C + 1).isPowerOf2()) | ||||||||
| 1614 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); | ||||||||
| 1615 | } | ||||||||
| 1616 | if (Pred == ICmpInst::ICMP_ULT) { | ||||||||
| 1617 | // (xor X, -C) <u C --> X >u ~C (when C is a power of 2) | ||||||||
| 1618 | if (*XorC == -C && C.isPowerOf2()) | ||||||||
| 1619 | return new ICmpInst(ICmpInst::ICMP_UGT, X, | ||||||||
| 1620 | ConstantInt::get(X->getType(), ~C)); | ||||||||
| 1621 | // (xor X, C) <u C --> X >u ~C (when -C is a power of 2) | ||||||||
| 1622 | if (*XorC == C && (-C).isPowerOf2()) | ||||||||
| 1623 | return new ICmpInst(ICmpInst::ICMP_UGT, X, | ||||||||
| 1624 | ConstantInt::get(X->getType(), ~C)); | ||||||||
| 1625 | } | ||||||||
| 1626 | return nullptr; | ||||||||
| 1627 | } | ||||||||
| 1628 | |||||||||
| 1629 | /// Fold icmp (and (sh X, Y), C2), C1. | ||||||||
| 1630 | Instruction *InstCombiner::foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And, | ||||||||
| 1631 | const APInt &C1, const APInt &C2) { | ||||||||
| 1632 | BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0)); | ||||||||
| 1633 | if (!Shift || !Shift->isShift()) | ||||||||
| 1634 | return nullptr; | ||||||||
| 1635 | |||||||||
| 1636 | // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could | ||||||||
| 1637 | // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in | ||||||||
| 1638 | // code produced by the clang front-end, for bitfield access. | ||||||||
| 1639 | // This seemingly simple opportunity to fold away a shift turns out to be | ||||||||
| 1640 | // rather complicated. See PR17827 for details. | ||||||||
| 1641 | unsigned ShiftOpcode = Shift->getOpcode(); | ||||||||
| 1642 | bool IsShl = ShiftOpcode == Instruction::Shl; | ||||||||
| 1643 | const APInt *C3; | ||||||||
| 1644 | if (match(Shift->getOperand(1), m_APInt(C3))) { | ||||||||
| 1645 | APInt NewAndCst, NewCmpCst; | ||||||||
| 1646 | bool AnyCmpCstBitsShiftedOut; | ||||||||
| 1647 | if (ShiftOpcode == Instruction::Shl) { | ||||||||
| 1648 | // For a left shift, we can fold if the comparison is not signed. We can | ||||||||
| 1649 | // also fold a signed comparison if the mask value and comparison value | ||||||||
| 1650 | // are not negative. These constraints may not be obvious, but we can | ||||||||
| 1651 | // prove that they are correct using an SMT solver. | ||||||||
| 1652 | if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative())) | ||||||||
| 1653 | return nullptr; | ||||||||
| 1654 | |||||||||
| 1655 | NewCmpCst = C1.lshr(*C3); | ||||||||
| 1656 | NewAndCst = C2.lshr(*C3); | ||||||||
| 1657 | AnyCmpCstBitsShiftedOut = NewCmpCst.shl(*C3) != C1; | ||||||||
| 1658 | } else if (ShiftOpcode == Instruction::LShr) { | ||||||||
| 1659 | // For a logical right shift, we can fold if the comparison is not signed. | ||||||||
| 1660 | // We can also fold a signed comparison if the shifted mask value and the | ||||||||
| 1661 | // shifted comparison value are not negative. These constraints may not be | ||||||||
| 1662 | // obvious, but we can prove that they are correct using an SMT solver. | ||||||||
| 1663 | NewCmpCst = C1.shl(*C3); | ||||||||
| 1664 | NewAndCst = C2.shl(*C3); | ||||||||
| 1665 | AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(*C3) != C1; | ||||||||
| 1666 | if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative())) | ||||||||
| 1667 | return nullptr; | ||||||||
| 1668 | } else { | ||||||||
| 1669 | // For an arithmetic shift, check that both constants don't use (in a | ||||||||
| 1670 | // signed sense) the top bits being shifted out. | ||||||||
| 1671 | assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode")((ShiftOpcode == Instruction::AShr && "Unknown shift opcode" ) ? static_cast<void> (0) : __assert_fail ("ShiftOpcode == Instruction::AShr && \"Unknown shift opcode\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 1671, __PRETTY_FUNCTION__)); | ||||||||
| 1672 | NewCmpCst = C1.shl(*C3); | ||||||||
| 1673 | NewAndCst = C2.shl(*C3); | ||||||||
| 1674 | AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(*C3) != C1; | ||||||||
| 1675 | if (NewAndCst.ashr(*C3) != C2) | ||||||||
| 1676 | return nullptr; | ||||||||
| 1677 | } | ||||||||
| 1678 | |||||||||
| 1679 | if (AnyCmpCstBitsShiftedOut) { | ||||||||
| 1680 | // If we shifted bits out, the fold is not going to work out. As a | ||||||||
| 1681 | // special case, check to see if this means that the result is always | ||||||||
| 1682 | // true or false now. | ||||||||
| 1683 | if (Cmp.getPredicate() == ICmpInst::ICMP_EQ) | ||||||||
| 1684 | return replaceInstUsesWith(Cmp, ConstantInt::getFalse(Cmp.getType())); | ||||||||
| 1685 | if (Cmp.getPredicate() == ICmpInst::ICMP_NE) | ||||||||
| 1686 | return replaceInstUsesWith(Cmp, ConstantInt::getTrue(Cmp.getType())); | ||||||||
| 1687 | } else { | ||||||||
| 1688 | Value *NewAnd = Builder.CreateAnd( | ||||||||
| 1689 | Shift->getOperand(0), ConstantInt::get(And->getType(), NewAndCst)); | ||||||||
| 1690 | return new ICmpInst(Cmp.getPredicate(), | ||||||||
| 1691 | NewAnd, ConstantInt::get(And->getType(), NewCmpCst)); | ||||||||
| 1692 | } | ||||||||
| 1693 | } | ||||||||
| 1694 | |||||||||
| 1695 | // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is | ||||||||
| 1696 | // preferable because it allows the C2 << Y expression to be hoisted out of a | ||||||||
| 1697 | // loop if Y is invariant and X is not. | ||||||||
| 1698 | if (Shift->hasOneUse() && C1.isNullValue() && Cmp.isEquality() && | ||||||||
| 1699 | !Shift->isArithmeticShift() && !isa<Constant>(Shift->getOperand(0))) { | ||||||||
| 1700 | // Compute C2 << Y. | ||||||||
| 1701 | Value *NewShift = | ||||||||
| 1702 | IsShl ? Builder.CreateLShr(And->getOperand(1), Shift->getOperand(1)) | ||||||||
| 1703 | : Builder.CreateShl(And->getOperand(1), Shift->getOperand(1)); | ||||||||
| 1704 | |||||||||
| 1705 | // Compute X & (C2 << Y). | ||||||||
| 1706 | Value *NewAnd = Builder.CreateAnd(Shift->getOperand(0), NewShift); | ||||||||
| 1707 | return replaceOperand(Cmp, 0, NewAnd); | ||||||||
| 1708 | } | ||||||||
| 1709 | |||||||||
| 1710 | return nullptr; | ||||||||
| 1711 | } | ||||||||
| 1712 | |||||||||
| 1713 | /// Fold icmp (and X, C2), C1. | ||||||||
| 1714 | Instruction *InstCombiner::foldICmpAndConstConst(ICmpInst &Cmp, | ||||||||
| 1715 | BinaryOperator *And, | ||||||||
| 1716 | const APInt &C1) { | ||||||||
| 1717 | bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE; | ||||||||
| 1718 | |||||||||
| 1719 | // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1 | ||||||||
| 1720 | // TODO: We canonicalize to the longer form for scalars because we have | ||||||||
| 1721 | // better analysis/folds for icmp, and codegen may be better with icmp. | ||||||||
| 1722 | if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isNullValue() && | ||||||||
| 1723 | match(And->getOperand(1), m_One())) | ||||||||
| 1724 | return new TruncInst(And->getOperand(0), Cmp.getType()); | ||||||||
| 1725 | |||||||||
| 1726 | const APInt *C2; | ||||||||
| 1727 | Value *X; | ||||||||
| 1728 | if (!match(And, m_And(m_Value(X), m_APInt(C2)))) | ||||||||
| 1729 | return nullptr; | ||||||||
| 1730 | |||||||||
| 1731 | // Don't perform the following transforms if the AND has multiple uses | ||||||||
| 1732 | if (!And->hasOneUse()) | ||||||||
| 1733 | return nullptr; | ||||||||
| 1734 | |||||||||
| 1735 | if (Cmp.isEquality() && C1.isNullValue()) { | ||||||||
| 1736 | // Restrict this fold to single-use 'and' (PR10267). | ||||||||
| 1737 | // Replace (and X, (1 << size(X)-1) != 0) with X s< 0 | ||||||||
| 1738 | if (C2->isSignMask()) { | ||||||||
| 1739 | Constant *Zero = Constant::getNullValue(X->getType()); | ||||||||
| 1740 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE; | ||||||||
| 1741 | return new ICmpInst(NewPred, X, Zero); | ||||||||
| 1742 | } | ||||||||
| 1743 | |||||||||
| 1744 | // Restrict this fold only for single-use 'and' (PR10267). | ||||||||
| 1745 | // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two. | ||||||||
| 1746 | if ((~(*C2) + 1).isPowerOf2()) { | ||||||||
| 1747 | Constant *NegBOC = | ||||||||
| 1748 | ConstantExpr::getNeg(cast<Constant>(And->getOperand(1))); | ||||||||
| 1749 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; | ||||||||
| 1750 | return new ICmpInst(NewPred, X, NegBOC); | ||||||||
| 1751 | } | ||||||||
| 1752 | } | ||||||||
| 1753 | |||||||||
| 1754 | // If the LHS is an 'and' of a truncate and we can widen the and/compare to | ||||||||
| 1755 | // the input width without changing the value produced, eliminate the cast: | ||||||||
| 1756 | // | ||||||||
| 1757 | // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1' | ||||||||
| 1758 | // | ||||||||
| 1759 | // We can do this transformation if the constants do not have their sign bits | ||||||||
| 1760 | // set or if it is an equality comparison. Extending a relational comparison | ||||||||
| 1761 | // when we're checking the sign bit would not work. | ||||||||
| 1762 | Value *W; | ||||||||
| 1763 | if (match(And->getOperand(0), m_OneUse(m_Trunc(m_Value(W)))) && | ||||||||
| 1764 | (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) { | ||||||||
| 1765 | // TODO: Is this a good transform for vectors? Wider types may reduce | ||||||||
| 1766 | // throughput. Should this transform be limited (even for scalars) by using | ||||||||
| 1767 | // shouldChangeType()? | ||||||||
| 1768 | if (!Cmp.getType()->isVectorTy()) { | ||||||||
| 1769 | Type *WideType = W->getType(); | ||||||||
| 1770 | unsigned WideScalarBits = WideType->getScalarSizeInBits(); | ||||||||
| 1771 | Constant *ZextC1 = ConstantInt::get(WideType, C1.zext(WideScalarBits)); | ||||||||
| 1772 | Constant *ZextC2 = ConstantInt::get(WideType, C2->zext(WideScalarBits)); | ||||||||
| 1773 | Value *NewAnd = Builder.CreateAnd(W, ZextC2, And->getName()); | ||||||||
| 1774 | return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1); | ||||||||
| 1775 | } | ||||||||
| 1776 | } | ||||||||
| 1777 | |||||||||
| 1778 | if (Instruction *I = foldICmpAndShift(Cmp, And, C1, *C2)) | ||||||||
| 1779 | return I; | ||||||||
| 1780 | |||||||||
| 1781 | // (icmp pred (and (or (lshr A, B), A), 1), 0) --> | ||||||||
| 1782 | // (icmp pred (and A, (or (shl 1, B), 1), 0)) | ||||||||
| 1783 | // | ||||||||
| 1784 | // iff pred isn't signed | ||||||||
| 1785 | if (!Cmp.isSigned() && C1.isNullValue() && And->getOperand(0)->hasOneUse() && | ||||||||
| 1786 | match(And->getOperand(1), m_One())) { | ||||||||
| 1787 | Constant *One = cast<Constant>(And->getOperand(1)); | ||||||||
| 1788 | Value *Or = And->getOperand(0); | ||||||||
| 1789 | Value *A, *B, *LShr; | ||||||||
| 1790 | if (match(Or, m_Or(m_Value(LShr), m_Value(A))) && | ||||||||
| 1791 | match(LShr, m_LShr(m_Specific(A), m_Value(B)))) { | ||||||||
| 1792 | unsigned UsesRemoved = 0; | ||||||||
| 1793 | if (And->hasOneUse()) | ||||||||
| 1794 | ++UsesRemoved; | ||||||||
| 1795 | if (Or->hasOneUse()) | ||||||||
| 1796 | ++UsesRemoved; | ||||||||
| 1797 | if (LShr->hasOneUse()) | ||||||||
| 1798 | ++UsesRemoved; | ||||||||
| 1799 | |||||||||
| 1800 | // Compute A & ((1 << B) | 1) | ||||||||
| 1801 | Value *NewOr = nullptr; | ||||||||
| 1802 | if (auto *C = dyn_cast<Constant>(B)) { | ||||||||
| 1803 | if (UsesRemoved >= 1) | ||||||||
| 1804 | NewOr = ConstantExpr::getOr(ConstantExpr::getNUWShl(One, C), One); | ||||||||
| 1805 | } else { | ||||||||
| 1806 | if (UsesRemoved >= 3) | ||||||||
| 1807 | NewOr = Builder.CreateOr(Builder.CreateShl(One, B, LShr->getName(), | ||||||||
| 1808 | /*HasNUW=*/true), | ||||||||
| 1809 | One, Or->getName()); | ||||||||
| 1810 | } | ||||||||
| 1811 | if (NewOr) { | ||||||||
| 1812 | Value *NewAnd = Builder.CreateAnd(A, NewOr, And->getName()); | ||||||||
| 1813 | return replaceOperand(Cmp, 0, NewAnd); | ||||||||
| 1814 | } | ||||||||
| 1815 | } | ||||||||
| 1816 | } | ||||||||
| 1817 | |||||||||
| 1818 | return nullptr; | ||||||||
| 1819 | } | ||||||||
| 1820 | |||||||||
| 1821 | /// Fold icmp (and X, Y), C. | ||||||||
| 1822 | Instruction *InstCombiner::foldICmpAndConstant(ICmpInst &Cmp, | ||||||||
| 1823 | BinaryOperator *And, | ||||||||
| 1824 | const APInt &C) { | ||||||||
| 1825 | if (Instruction *I = foldICmpAndConstConst(Cmp, And, C)) | ||||||||
| 1826 | return I; | ||||||||
| 1827 | |||||||||
| 1828 | // TODO: These all require that Y is constant too, so refactor with the above. | ||||||||
| 1829 | |||||||||
| 1830 | // Try to optimize things like "A[i] & 42 == 0" to index computations. | ||||||||
| 1831 | Value *X = And->getOperand(0); | ||||||||
| 1832 | Value *Y = And->getOperand(1); | ||||||||
| 1833 | if (auto *LI = dyn_cast<LoadInst>(X)) | ||||||||
| 1834 | if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) | ||||||||
| 1835 | if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||||||
| 1836 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||||||
| 1837 | !LI->isVolatile() && isa<ConstantInt>(Y)) { | ||||||||
| 1838 | ConstantInt *C2 = cast<ConstantInt>(Y); | ||||||||
| 1839 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, Cmp, C2)) | ||||||||
| 1840 | return Res; | ||||||||
| 1841 | } | ||||||||
| 1842 | |||||||||
| 1843 | if (!Cmp.isEquality()) | ||||||||
| 1844 | return nullptr; | ||||||||
| 1845 | |||||||||
| 1846 | // X & -C == -C -> X > u ~C | ||||||||
| 1847 | // X & -C != -C -> X <= u ~C | ||||||||
| 1848 | // iff C is a power of 2 | ||||||||
| 1849 | if (Cmp.getOperand(1) == Y && (-C).isPowerOf2()) { | ||||||||
| 1850 | auto NewPred = Cmp.getPredicate() == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT | ||||||||
| 1851 | : CmpInst::ICMP_ULE; | ||||||||
| 1852 | return new ICmpInst(NewPred, X, SubOne(cast<Constant>(Cmp.getOperand(1)))); | ||||||||
| 1853 | } | ||||||||
| 1854 | |||||||||
| 1855 | // (X & C2) == 0 -> (trunc X) >= 0 | ||||||||
| 1856 | // (X & C2) != 0 -> (trunc X) < 0 | ||||||||
| 1857 | // iff C2 is a power of 2 and it masks the sign bit of a legal integer type. | ||||||||
| 1858 | const APInt *C2; | ||||||||
| 1859 | if (And->hasOneUse() && C.isNullValue() && match(Y, m_APInt(C2))) { | ||||||||
| 1860 | int32_t ExactLogBase2 = C2->exactLogBase2(); | ||||||||
| 1861 | if (ExactLogBase2 != -1 && DL.isLegalInteger(ExactLogBase2 + 1)) { | ||||||||
| 1862 | Type *NTy = IntegerType::get(Cmp.getContext(), ExactLogBase2 + 1); | ||||||||
| 1863 | if (And->getType()->isVectorTy()) | ||||||||
| 1864 | NTy = VectorType::get(NTy, And->getType()->getVectorNumElements()); | ||||||||
| 1865 | Value *Trunc = Builder.CreateTrunc(X, NTy); | ||||||||
| 1866 | auto NewPred = Cmp.getPredicate() == CmpInst::ICMP_EQ ? CmpInst::ICMP_SGE | ||||||||
| 1867 | : CmpInst::ICMP_SLT; | ||||||||
| 1868 | return new ICmpInst(NewPred, Trunc, Constant::getNullValue(NTy)); | ||||||||
| 1869 | } | ||||||||
| 1870 | } | ||||||||
| 1871 | |||||||||
| 1872 | return nullptr; | ||||||||
| 1873 | } | ||||||||
| 1874 | |||||||||
| 1875 | /// Fold icmp (or X, Y), C. | ||||||||
| 1876 | Instruction *InstCombiner::foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or, | ||||||||
| 1877 | const APInt &C) { | ||||||||
| 1878 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1879 | if (C.isOneValue()) { | ||||||||
| 1880 | // icmp slt signum(V) 1 --> icmp slt V, 1 | ||||||||
| 1881 | Value *V = nullptr; | ||||||||
| 1882 | if (Pred == ICmpInst::ICMP_SLT && match(Or, m_Signum(m_Value(V)))) | ||||||||
| 1883 | return new ICmpInst(ICmpInst::ICMP_SLT, V, | ||||||||
| 1884 | ConstantInt::get(V->getType(), 1)); | ||||||||
| 1885 | } | ||||||||
| 1886 | |||||||||
| 1887 | Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1); | ||||||||
| 1888 | if (Cmp.isEquality() && Cmp.getOperand(1) == OrOp1) { | ||||||||
| 1889 | // X | C == C --> X <=u C | ||||||||
| 1890 | // X | C != C --> X >u C | ||||||||
| 1891 | // iff C+1 is a power of 2 (C is a bitmask of the low bits) | ||||||||
| 1892 | if ((C + 1).isPowerOf2()) { | ||||||||
| 1893 | Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT; | ||||||||
| 1894 | return new ICmpInst(Pred, OrOp0, OrOp1); | ||||||||
| 1895 | } | ||||||||
| 1896 | // More general: are all bits outside of a mask constant set or not set? | ||||||||
| 1897 | // X | C == C --> (X & ~C) == 0 | ||||||||
| 1898 | // X | C != C --> (X & ~C) != 0 | ||||||||
| 1899 | if (Or->hasOneUse()) { | ||||||||
| 1900 | Value *A = Builder.CreateAnd(OrOp0, ~C); | ||||||||
| 1901 | return new ICmpInst(Pred, A, ConstantInt::getNullValue(OrOp0->getType())); | ||||||||
| 1902 | } | ||||||||
| 1903 | } | ||||||||
| 1904 | |||||||||
| 1905 | if (!Cmp.isEquality() || !C.isNullValue() || !Or->hasOneUse()) | ||||||||
| 1906 | return nullptr; | ||||||||
| 1907 | |||||||||
| 1908 | Value *P, *Q; | ||||||||
| 1909 | if (match(Or, m_Or(m_PtrToInt(m_Value(P)), m_PtrToInt(m_Value(Q))))) { | ||||||||
| 1910 | // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0 | ||||||||
| 1911 | // -> and (icmp eq P, null), (icmp eq Q, null). | ||||||||
| 1912 | Value *CmpP = | ||||||||
| 1913 | Builder.CreateICmp(Pred, P, ConstantInt::getNullValue(P->getType())); | ||||||||
| 1914 | Value *CmpQ = | ||||||||
| 1915 | Builder.CreateICmp(Pred, Q, ConstantInt::getNullValue(Q->getType())); | ||||||||
| 1916 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; | ||||||||
| 1917 | return BinaryOperator::Create(BOpc, CmpP, CmpQ); | ||||||||
| 1918 | } | ||||||||
| 1919 | |||||||||
| 1920 | // Are we using xors to bitwise check for a pair of (in)equalities? Convert to | ||||||||
| 1921 | // a shorter form that has more potential to be folded even further. | ||||||||
| 1922 | Value *X1, *X2, *X3, *X4; | ||||||||
| 1923 | if (match(OrOp0, m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) && | ||||||||
| 1924 | match(OrOp1, m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) { | ||||||||
| 1925 | // ((X1 ^ X2) || (X3 ^ X4)) == 0 --> (X1 == X2) && (X3 == X4) | ||||||||
| 1926 | // ((X1 ^ X2) || (X3 ^ X4)) != 0 --> (X1 != X2) || (X3 != X4) | ||||||||
| 1927 | Value *Cmp12 = Builder.CreateICmp(Pred, X1, X2); | ||||||||
| 1928 | Value *Cmp34 = Builder.CreateICmp(Pred, X3, X4); | ||||||||
| 1929 | auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; | ||||||||
| 1930 | return BinaryOperator::Create(BOpc, Cmp12, Cmp34); | ||||||||
| 1931 | } | ||||||||
| 1932 | |||||||||
| 1933 | return nullptr; | ||||||||
| 1934 | } | ||||||||
| 1935 | |||||||||
| 1936 | /// Fold icmp (mul X, Y), C. | ||||||||
| 1937 | Instruction *InstCombiner::foldICmpMulConstant(ICmpInst &Cmp, | ||||||||
| 1938 | BinaryOperator *Mul, | ||||||||
| 1939 | const APInt &C) { | ||||||||
| 1940 | const APInt *MulC; | ||||||||
| 1941 | if (!match(Mul->getOperand(1), m_APInt(MulC))) | ||||||||
| 1942 | return nullptr; | ||||||||
| 1943 | |||||||||
| 1944 | // If this is a test of the sign bit and the multiply is sign-preserving with | ||||||||
| 1945 | // a constant operand, use the multiply LHS operand instead. | ||||||||
| 1946 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1947 | if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) { | ||||||||
| 1948 | if (MulC->isNegative()) | ||||||||
| 1949 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||||||
| 1950 | return new ICmpInst(Pred, Mul->getOperand(0), | ||||||||
| 1951 | Constant::getNullValue(Mul->getType())); | ||||||||
| 1952 | } | ||||||||
| 1953 | |||||||||
| 1954 | return nullptr; | ||||||||
| 1955 | } | ||||||||
| 1956 | |||||||||
| 1957 | /// Fold icmp (shl 1, Y), C. | ||||||||
| 1958 | static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl, | ||||||||
| 1959 | const APInt &C) { | ||||||||
| 1960 | Value *Y; | ||||||||
| 1961 | if (!match(Shl, m_Shl(m_One(), m_Value(Y)))) | ||||||||
| 1962 | return nullptr; | ||||||||
| 1963 | |||||||||
| 1964 | Type *ShiftType = Shl->getType(); | ||||||||
| 1965 | unsigned TypeBits = C.getBitWidth(); | ||||||||
| 1966 | bool CIsPowerOf2 = C.isPowerOf2(); | ||||||||
| 1967 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 1968 | if (Cmp.isUnsigned()) { | ||||||||
| 1969 | // (1 << Y) pred C -> Y pred Log2(C) | ||||||||
| 1970 | if (!CIsPowerOf2) { | ||||||||
| 1971 | // (1 << Y) < 30 -> Y <= 4 | ||||||||
| 1972 | // (1 << Y) <= 30 -> Y <= 4 | ||||||||
| 1973 | // (1 << Y) >= 30 -> Y > 4 | ||||||||
| 1974 | // (1 << Y) > 30 -> Y > 4 | ||||||||
| 1975 | if (Pred == ICmpInst::ICMP_ULT) | ||||||||
| 1976 | Pred = ICmpInst::ICMP_ULE; | ||||||||
| 1977 | else if (Pred == ICmpInst::ICMP_UGE) | ||||||||
| 1978 | Pred = ICmpInst::ICMP_UGT; | ||||||||
| 1979 | } | ||||||||
| 1980 | |||||||||
| 1981 | // (1 << Y) >= 2147483648 -> Y >= 31 -> Y == 31 | ||||||||
| 1982 | // (1 << Y) < 2147483648 -> Y < 31 -> Y != 31 | ||||||||
| 1983 | unsigned CLog2 = C.logBase2(); | ||||||||
| 1984 | if (CLog2 == TypeBits - 1) { | ||||||||
| 1985 | if (Pred == ICmpInst::ICMP_UGE) | ||||||||
| 1986 | Pred = ICmpInst::ICMP_EQ; | ||||||||
| 1987 | else if (Pred == ICmpInst::ICMP_ULT) | ||||||||
| 1988 | Pred = ICmpInst::ICMP_NE; | ||||||||
| 1989 | } | ||||||||
| 1990 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, CLog2)); | ||||||||
| 1991 | } else if (Cmp.isSigned()) { | ||||||||
| 1992 | Constant *BitWidthMinusOne = ConstantInt::get(ShiftType, TypeBits - 1); | ||||||||
| 1993 | if (C.isAllOnesValue()) { | ||||||||
| 1994 | // (1 << Y) <= -1 -> Y == 31 | ||||||||
| 1995 | if (Pred == ICmpInst::ICMP_SLE) | ||||||||
| 1996 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | ||||||||
| 1997 | |||||||||
| 1998 | // (1 << Y) > -1 -> Y != 31 | ||||||||
| 1999 | if (Pred == ICmpInst::ICMP_SGT) | ||||||||
| 2000 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | ||||||||
| 2001 | } else if (!C) { | ||||||||
| 2002 | // (1 << Y) < 0 -> Y == 31 | ||||||||
| 2003 | // (1 << Y) <= 0 -> Y == 31 | ||||||||
| 2004 | if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) | ||||||||
| 2005 | return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne); | ||||||||
| 2006 | |||||||||
| 2007 | // (1 << Y) >= 0 -> Y != 31 | ||||||||
| 2008 | // (1 << Y) > 0 -> Y != 31 | ||||||||
| 2009 | if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) | ||||||||
| 2010 | return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne); | ||||||||
| 2011 | } | ||||||||
| 2012 | } else if (Cmp.isEquality() && CIsPowerOf2) { | ||||||||
| 2013 | return new ICmpInst(Pred, Y, ConstantInt::get(ShiftType, C.logBase2())); | ||||||||
| 2014 | } | ||||||||
| 2015 | |||||||||
| 2016 | return nullptr; | ||||||||
| 2017 | } | ||||||||
| 2018 | |||||||||
| 2019 | /// Fold icmp (shl X, Y), C. | ||||||||
| 2020 | Instruction *InstCombiner::foldICmpShlConstant(ICmpInst &Cmp, | ||||||||
| 2021 | BinaryOperator *Shl, | ||||||||
| 2022 | const APInt &C) { | ||||||||
| 2023 | const APInt *ShiftVal; | ||||||||
| 2024 | if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal))) | ||||||||
| 2025 | return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal); | ||||||||
| 2026 | |||||||||
| 2027 | const APInt *ShiftAmt; | ||||||||
| 2028 | if (!match(Shl->getOperand(1), m_APInt(ShiftAmt))) | ||||||||
| 2029 | return foldICmpShlOne(Cmp, Shl, C); | ||||||||
| 2030 | |||||||||
| 2031 | // Check that the shift amount is in range. If not, don't perform undefined | ||||||||
| 2032 | // shifts. When the shift is visited, it will be simplified. | ||||||||
| 2033 | unsigned TypeBits = C.getBitWidth(); | ||||||||
| 2034 | if (ShiftAmt->uge(TypeBits)) | ||||||||
| 2035 | return nullptr; | ||||||||
| 2036 | |||||||||
| 2037 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2038 | Value *X = Shl->getOperand(0); | ||||||||
| 2039 | Type *ShType = Shl->getType(); | ||||||||
| 2040 | |||||||||
| 2041 | // NSW guarantees that we are only shifting out sign bits from the high bits, | ||||||||
| 2042 | // so we can ASHR the compare constant without needing a mask and eliminate | ||||||||
| 2043 | // the shift. | ||||||||
| 2044 | if (Shl->hasNoSignedWrap()) { | ||||||||
| 2045 | if (Pred == ICmpInst::ICMP_SGT) { | ||||||||
| 2046 | // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt) | ||||||||
| 2047 | APInt ShiftedC = C.ashr(*ShiftAmt); | ||||||||
| 2048 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
| 2049 | } | ||||||||
| 2050 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && | ||||||||
| 2051 | C.ashr(*ShiftAmt).shl(*ShiftAmt) == C) { | ||||||||
| 2052 | APInt ShiftedC = C.ashr(*ShiftAmt); | ||||||||
| 2053 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
| 2054 | } | ||||||||
| 2055 | if (Pred == ICmpInst::ICMP_SLT) { | ||||||||
| 2056 | // SLE is the same as above, but SLE is canonicalized to SLT, so convert: | ||||||||
| 2057 | // (X << S) <=s C is equiv to X <=s (C >> S) for all C | ||||||||
| 2058 | // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX | ||||||||
| 2059 | // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN | ||||||||
| 2060 | assert(!C.isMinSignedValue() && "Unexpected icmp slt")((!C.isMinSignedValue() && "Unexpected icmp slt") ? static_cast <void> (0) : __assert_fail ("!C.isMinSignedValue() && \"Unexpected icmp slt\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2060, __PRETTY_FUNCTION__)); | ||||||||
| 2061 | APInt ShiftedC = (C - 1).ashr(*ShiftAmt) + 1; | ||||||||
| 2062 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
| 2063 | } | ||||||||
| 2064 | // If this is a signed comparison to 0 and the shift is sign preserving, | ||||||||
| 2065 | // use the shift LHS operand instead; isSignTest may change 'Pred', so only | ||||||||
| 2066 | // do that if we're sure to not continue on in this function. | ||||||||
| 2067 | if (isSignTest(Pred, C)) | ||||||||
| 2068 | return new ICmpInst(Pred, X, Constant::getNullValue(ShType)); | ||||||||
| 2069 | } | ||||||||
| 2070 | |||||||||
| 2071 | // NUW guarantees that we are only shifting out zero bits from the high bits, | ||||||||
| 2072 | // so we can LSHR the compare constant without needing a mask and eliminate | ||||||||
| 2073 | // the shift. | ||||||||
| 2074 | if (Shl->hasNoUnsignedWrap()) { | ||||||||
| 2075 | if (Pred == ICmpInst::ICMP_UGT) { | ||||||||
| 2076 | // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt) | ||||||||
| 2077 | APInt ShiftedC = C.lshr(*ShiftAmt); | ||||||||
| 2078 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
| 2079 | } | ||||||||
| 2080 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && | ||||||||
| 2081 | C.lshr(*ShiftAmt).shl(*ShiftAmt) == C) { | ||||||||
| 2082 | APInt ShiftedC = C.lshr(*ShiftAmt); | ||||||||
| 2083 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
| 2084 | } | ||||||||
| 2085 | if (Pred == ICmpInst::ICMP_ULT) { | ||||||||
| 2086 | // ULE is the same as above, but ULE is canonicalized to ULT, so convert: | ||||||||
| 2087 | // (X << S) <=u C is equiv to X <=u (C >> S) for all C | ||||||||
| 2088 | // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u | ||||||||
| 2089 | // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0 | ||||||||
| 2090 | assert(C.ugt(0) && "ult 0 should have been eliminated")((C.ugt(0) && "ult 0 should have been eliminated") ? static_cast <void> (0) : __assert_fail ("C.ugt(0) && \"ult 0 should have been eliminated\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2090, __PRETTY_FUNCTION__)); | ||||||||
| 2091 | APInt ShiftedC = (C - 1).lshr(*ShiftAmt) + 1; | ||||||||
| 2092 | return new ICmpInst(Pred, X, ConstantInt::get(ShType, ShiftedC)); | ||||||||
| 2093 | } | ||||||||
| 2094 | } | ||||||||
| 2095 | |||||||||
| 2096 | if (Cmp.isEquality() && Shl->hasOneUse()) { | ||||||||
| 2097 | // Strength-reduce the shift into an 'and'. | ||||||||
| 2098 | Constant *Mask = ConstantInt::get( | ||||||||
| 2099 | ShType, | ||||||||
| 2100 | APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt->getZExtValue())); | ||||||||
| 2101 | Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); | ||||||||
| 2102 | Constant *LShrC = ConstantInt::get(ShType, C.lshr(*ShiftAmt)); | ||||||||
| 2103 | return new ICmpInst(Pred, And, LShrC); | ||||||||
| 2104 | } | ||||||||
| 2105 | |||||||||
| 2106 | // Otherwise, if this is a comparison of the sign bit, simplify to and/test. | ||||||||
| 2107 | bool TrueIfSigned = false; | ||||||||
| 2108 | if (Shl->hasOneUse() && isSignBitCheck(Pred, C, TrueIfSigned)) { | ||||||||
| 2109 | // (X << 31) <s 0 --> (X & 1) != 0 | ||||||||
| 2110 | Constant *Mask = ConstantInt::get( | ||||||||
| 2111 | ShType, | ||||||||
| 2112 | APInt::getOneBitSet(TypeBits, TypeBits - ShiftAmt->getZExtValue() - 1)); | ||||||||
| 2113 | Value *And = Builder.CreateAnd(X, Mask, Shl->getName() + ".mask"); | ||||||||
| 2114 | return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ, | ||||||||
| 2115 | And, Constant::getNullValue(ShType)); | ||||||||
| 2116 | } | ||||||||
| 2117 | |||||||||
| 2118 | // Simplify 'shl' inequality test into 'and' equality test. | ||||||||
| 2119 | if (Cmp.isUnsigned() && Shl->hasOneUse()) { | ||||||||
| 2120 | // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0 | ||||||||
| 2121 | if ((C + 1).isPowerOf2() && | ||||||||
| 2122 | (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) { | ||||||||
| 2123 | Value *And = Builder.CreateAnd(X, (~C).lshr(ShiftAmt->getZExtValue())); | ||||||||
| 2124 | return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ | ||||||||
| 2125 | : ICmpInst::ICMP_NE, | ||||||||
| 2126 | And, Constant::getNullValue(ShType)); | ||||||||
| 2127 | } | ||||||||
| 2128 | // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0 | ||||||||
| 2129 | if (C.isPowerOf2() && | ||||||||
| 2130 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) { | ||||||||
| 2131 | Value *And = | ||||||||
| 2132 | Builder.CreateAnd(X, (~(C - 1)).lshr(ShiftAmt->getZExtValue())); | ||||||||
| 2133 | return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ | ||||||||
| 2134 | : ICmpInst::ICMP_NE, | ||||||||
| 2135 | And, Constant::getNullValue(ShType)); | ||||||||
| 2136 | } | ||||||||
| 2137 | } | ||||||||
| 2138 | |||||||||
| 2139 | // Transform (icmp pred iM (shl iM %v, N), C) | ||||||||
| 2140 | // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N)) | ||||||||
| 2141 | // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N. | ||||||||
| 2142 | // This enables us to get rid of the shift in favor of a trunc that may be | ||||||||
| 2143 | // free on the target. It has the additional benefit of comparing to a | ||||||||
| 2144 | // smaller constant that may be more target-friendly. | ||||||||
| 2145 | unsigned Amt = ShiftAmt->getLimitedValue(TypeBits - 1); | ||||||||
| 2146 | if (Shl->hasOneUse() && Amt != 0 && C.countTrailingZeros() >= Amt && | ||||||||
| 2147 | DL.isLegalInteger(TypeBits - Amt)) { | ||||||||
| 2148 | Type *TruncTy = IntegerType::get(Cmp.getContext(), TypeBits - Amt); | ||||||||
| 2149 | if (ShType->isVectorTy()) | ||||||||
| 2150 | TruncTy = VectorType::get(TruncTy, ShType->getVectorNumElements()); | ||||||||
| 2151 | Constant *NewC = | ||||||||
| 2152 | ConstantInt::get(TruncTy, C.ashr(*ShiftAmt).trunc(TypeBits - Amt)); | ||||||||
| 2153 | return new ICmpInst(Pred, Builder.CreateTrunc(X, TruncTy), NewC); | ||||||||
| 2154 | } | ||||||||
| 2155 | |||||||||
| 2156 | return nullptr; | ||||||||
| 2157 | } | ||||||||
| 2158 | |||||||||
| 2159 | /// Fold icmp ({al}shr X, Y), C. | ||||||||
| 2160 | Instruction *InstCombiner::foldICmpShrConstant(ICmpInst &Cmp, | ||||||||
| 2161 | BinaryOperator *Shr, | ||||||||
| 2162 | const APInt &C) { | ||||||||
| 2163 | // An exact shr only shifts out zero bits, so: | ||||||||
| 2164 | // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0 | ||||||||
| 2165 | Value *X = Shr->getOperand(0); | ||||||||
| 2166 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2167 | if (Cmp.isEquality() && Shr->isExact() && Shr->hasOneUse() && | ||||||||
| 2168 | C.isNullValue()) | ||||||||
| 2169 | return new ICmpInst(Pred, X, Cmp.getOperand(1)); | ||||||||
| 2170 | |||||||||
| 2171 | const APInt *ShiftVal; | ||||||||
| 2172 | if (Cmp.isEquality() && match(Shr->getOperand(0), m_APInt(ShiftVal))) | ||||||||
| 2173 | return foldICmpShrConstConst(Cmp, Shr->getOperand(1), C, *ShiftVal); | ||||||||
| 2174 | |||||||||
| 2175 | const APInt *ShiftAmt; | ||||||||
| 2176 | if (!match(Shr->getOperand(1), m_APInt(ShiftAmt))) | ||||||||
| 2177 | return nullptr; | ||||||||
| 2178 | |||||||||
| 2179 | // Check that the shift amount is in range. If not, don't perform undefined | ||||||||
| 2180 | // shifts. When the shift is visited it will be simplified. | ||||||||
| 2181 | unsigned TypeBits = C.getBitWidth(); | ||||||||
| 2182 | unsigned ShAmtVal = ShiftAmt->getLimitedValue(TypeBits); | ||||||||
| 2183 | if (ShAmtVal >= TypeBits || ShAmtVal == 0) | ||||||||
| 2184 | return nullptr; | ||||||||
| 2185 | |||||||||
| 2186 | bool IsAShr = Shr->getOpcode() == Instruction::AShr; | ||||||||
| 2187 | bool IsExact = Shr->isExact(); | ||||||||
| 2188 | Type *ShrTy = Shr->getType(); | ||||||||
| 2189 | // TODO: If we could guarantee that InstSimplify would handle all of the | ||||||||
| 2190 | // constant-value-based preconditions in the folds below, then we could assert | ||||||||
| 2191 | // those conditions rather than checking them. This is difficult because of | ||||||||
| 2192 | // undef/poison (PR34838). | ||||||||
| 2193 | if (IsAShr) { | ||||||||
| 2194 | if (Pred == CmpInst::ICMP_SLT || (Pred == CmpInst::ICMP_SGT && IsExact)) { | ||||||||
| 2195 | // icmp slt (ashr X, ShAmtC), C --> icmp slt X, (C << ShAmtC) | ||||||||
| 2196 | // icmp sgt (ashr exact X, ShAmtC), C --> icmp sgt X, (C << ShAmtC) | ||||||||
| 2197 | APInt ShiftedC = C.shl(ShAmtVal); | ||||||||
| 2198 | if (ShiftedC.ashr(ShAmtVal) == C) | ||||||||
| 2199 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
| 2200 | } | ||||||||
| 2201 | if (Pred == CmpInst::ICMP_SGT) { | ||||||||
| 2202 | // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1 | ||||||||
| 2203 | APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; | ||||||||
| 2204 | if (!C.isMaxSignedValue() && !(C + 1).shl(ShAmtVal).isMinSignedValue() && | ||||||||
| 2205 | (ShiftedC + 1).ashr(ShAmtVal) == (C + 1)) | ||||||||
| 2206 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
| 2207 | } | ||||||||
| 2208 | } else { | ||||||||
| 2209 | if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) { | ||||||||
| 2210 | // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC) | ||||||||
| 2211 | // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC) | ||||||||
| 2212 | APInt ShiftedC = C.shl(ShAmtVal); | ||||||||
| 2213 | if (ShiftedC.lshr(ShAmtVal) == C) | ||||||||
| 2214 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
| 2215 | } | ||||||||
| 2216 | if (Pred == CmpInst::ICMP_UGT) { | ||||||||
| 2217 | // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1 | ||||||||
| 2218 | APInt ShiftedC = (C + 1).shl(ShAmtVal) - 1; | ||||||||
| 2219 | if ((ShiftedC + 1).lshr(ShAmtVal) == (C + 1)) | ||||||||
| 2220 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC)); | ||||||||
| 2221 | } | ||||||||
| 2222 | } | ||||||||
| 2223 | |||||||||
| 2224 | if (!Cmp.isEquality()) | ||||||||
| 2225 | return nullptr; | ||||||||
| 2226 | |||||||||
| 2227 | // Handle equality comparisons of shift-by-constant. | ||||||||
| 2228 | |||||||||
| 2229 | // If the comparison constant changes with the shift, the comparison cannot | ||||||||
| 2230 | // succeed (bits of the comparison constant cannot match the shifted value). | ||||||||
| 2231 | // This should be known by InstSimplify and already be folded to true/false. | ||||||||
| 2232 | assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||((((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur.") ? static_cast< void> (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2234, __PRETTY_FUNCTION__)) | ||||||||
| 2233 | (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&((((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur.") ? static_cast< void> (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2234, __PRETTY_FUNCTION__)) | ||||||||
| 2234 | "Expected icmp+shr simplify did not occur.")((((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && "Expected icmp+shr simplify did not occur.") ? static_cast< void> (0) : __assert_fail ("((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) || (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) && \"Expected icmp+shr simplify did not occur.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2234, __PRETTY_FUNCTION__)); | ||||||||
| 2235 | |||||||||
| 2236 | // If the bits shifted out are known zero, compare the unshifted value: | ||||||||
| 2237 | // (X & 4) >> 1 == 2 --> (X & 4) == 4. | ||||||||
| 2238 | if (Shr->isExact()) | ||||||||
| 2239 | return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, C << ShAmtVal)); | ||||||||
| 2240 | |||||||||
| 2241 | if (Shr->hasOneUse()) { | ||||||||
| 2242 | // Canonicalize the shift into an 'and': | ||||||||
| 2243 | // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt) | ||||||||
| 2244 | APInt Val(APInt::getHighBitsSet(TypeBits, TypeBits - ShAmtVal)); | ||||||||
| 2245 | Constant *Mask = ConstantInt::get(ShrTy, Val); | ||||||||
| 2246 | Value *And = Builder.CreateAnd(X, Mask, Shr->getName() + ".mask"); | ||||||||
| 2247 | return new ICmpInst(Pred, And, ConstantInt::get(ShrTy, C << ShAmtVal)); | ||||||||
| 2248 | } | ||||||||
| 2249 | |||||||||
| 2250 | return nullptr; | ||||||||
| 2251 | } | ||||||||
| 2252 | |||||||||
| 2253 | Instruction *InstCombiner::foldICmpSRemConstant(ICmpInst &Cmp, | ||||||||
| 2254 | BinaryOperator *SRem, | ||||||||
| 2255 | const APInt &C) { | ||||||||
| 2256 | // Match an 'is positive' or 'is negative' comparison of remainder by a | ||||||||
| 2257 | // constant power-of-2 value: | ||||||||
| 2258 | // (X % pow2C) sgt/slt 0 | ||||||||
| 2259 | const ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2260 | if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT) | ||||||||
| 2261 | return nullptr; | ||||||||
| 2262 | |||||||||
| 2263 | // TODO: The one-use check is standard because we do not typically want to | ||||||||
| 2264 | // create longer instruction sequences, but this might be a special-case | ||||||||
| 2265 | // because srem is not good for analysis or codegen. | ||||||||
| 2266 | if (!SRem->hasOneUse()) | ||||||||
| 2267 | return nullptr; | ||||||||
| 2268 | |||||||||
| 2269 | const APInt *DivisorC; | ||||||||
| 2270 | if (!C.isNullValue() || !match(SRem->getOperand(1), m_Power2(DivisorC))) | ||||||||
| 2271 | return nullptr; | ||||||||
| 2272 | |||||||||
| 2273 | // Mask off the sign bit and the modulo bits (low-bits). | ||||||||
| 2274 | Type *Ty = SRem->getType(); | ||||||||
| 2275 | APInt SignMask = APInt::getSignMask(Ty->getScalarSizeInBits()); | ||||||||
| 2276 | Constant *MaskC = ConstantInt::get(Ty, SignMask | (*DivisorC - 1)); | ||||||||
| 2277 | Value *And = Builder.CreateAnd(SRem->getOperand(0), MaskC); | ||||||||
| 2278 | |||||||||
| 2279 | // For 'is positive?' check that the sign-bit is clear and at least 1 masked | ||||||||
| 2280 | // bit is set. Example: | ||||||||
| 2281 | // (i8 X % 32) s> 0 --> (X & 159) s> 0 | ||||||||
| 2282 | if (Pred == ICmpInst::ICMP_SGT) | ||||||||
| 2283 | return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty)); | ||||||||
| 2284 | |||||||||
| 2285 | // For 'is negative?' check that the sign-bit is set and at least 1 masked | ||||||||
| 2286 | // bit is set. Example: | ||||||||
| 2287 | // (i16 X % 4) s< 0 --> (X & 32771) u> 32768 | ||||||||
| 2288 | return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, SignMask)); | ||||||||
| 2289 | } | ||||||||
| 2290 | |||||||||
| 2291 | /// Fold icmp (udiv X, Y), C. | ||||||||
| 2292 | Instruction *InstCombiner::foldICmpUDivConstant(ICmpInst &Cmp, | ||||||||
| 2293 | BinaryOperator *UDiv, | ||||||||
| 2294 | const APInt &C) { | ||||||||
| 2295 | const APInt *C2; | ||||||||
| 2296 | if (!match(UDiv->getOperand(0), m_APInt(C2))) | ||||||||
| 2297 | return nullptr; | ||||||||
| 2298 | |||||||||
| 2299 | assert(*C2 != 0 && "udiv 0, X should have been simplified already.")((*C2 != 0 && "udiv 0, X should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("*C2 != 0 && \"udiv 0, X should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2299, __PRETTY_FUNCTION__)); | ||||||||
| 2300 | |||||||||
| 2301 | // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1)) | ||||||||
| 2302 | Value *Y = UDiv->getOperand(1); | ||||||||
| 2303 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT) { | ||||||||
| 2304 | assert(!C.isMaxValue() &&((!C.isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("!C.isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2305, __PRETTY_FUNCTION__)) | ||||||||
| 2305 | "icmp ugt X, UINT_MAX should have been simplified already.")((!C.isMaxValue() && "icmp ugt X, UINT_MAX should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("!C.isMaxValue() && \"icmp ugt X, UINT_MAX should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2305, __PRETTY_FUNCTION__)); | ||||||||
| 2306 | return new ICmpInst(ICmpInst::ICMP_ULE, Y, | ||||||||
| 2307 | ConstantInt::get(Y->getType(), C2->udiv(C + 1))); | ||||||||
| 2308 | } | ||||||||
| 2309 | |||||||||
| 2310 | // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C) | ||||||||
| 2311 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT) { | ||||||||
| 2312 | assert(C != 0 && "icmp ult X, 0 should have been simplified already.")((C != 0 && "icmp ult X, 0 should have been simplified already." ) ? static_cast<void> (0) : __assert_fail ("C != 0 && \"icmp ult X, 0 should have been simplified already.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2312, __PRETTY_FUNCTION__)); | ||||||||
| 2313 | return new ICmpInst(ICmpInst::ICMP_UGT, Y, | ||||||||
| 2314 | ConstantInt::get(Y->getType(), C2->udiv(C))); | ||||||||
| 2315 | } | ||||||||
| 2316 | |||||||||
| 2317 | return nullptr; | ||||||||
| 2318 | } | ||||||||
| 2319 | |||||||||
| 2320 | /// Fold icmp ({su}div X, Y), C. | ||||||||
| 2321 | Instruction *InstCombiner::foldICmpDivConstant(ICmpInst &Cmp, | ||||||||
| 2322 | BinaryOperator *Div, | ||||||||
| 2323 | const APInt &C) { | ||||||||
| 2324 | // Fold: icmp pred ([us]div X, C2), C -> range test | ||||||||
| 2325 | // Fold this div into the comparison, producing a range check. | ||||||||
| 2326 | // Determine, based on the divide type, what the range is being | ||||||||
| 2327 | // checked. If there is an overflow on the low or high side, remember | ||||||||
| 2328 | // it, otherwise compute the range [low, hi) bounding the new value. | ||||||||
| 2329 | // See: InsertRangeTest above for the kinds of replacements possible. | ||||||||
| 2330 | const APInt *C2; | ||||||||
| 2331 | if (!match(Div->getOperand(1), m_APInt(C2))) | ||||||||
| 2332 | return nullptr; | ||||||||
| 2333 | |||||||||
| 2334 | // FIXME: If the operand types don't match the type of the divide | ||||||||
| 2335 | // then don't attempt this transform. The code below doesn't have the | ||||||||
| 2336 | // logic to deal with a signed divide and an unsigned compare (and | ||||||||
| 2337 | // vice versa). This is because (x /s C2) <s C produces different | ||||||||
| 2338 | // results than (x /s C2) <u C or (x /u C2) <s C or even | ||||||||
| 2339 | // (x /u C2) <u C. Simply casting the operands and result won't | ||||||||
| 2340 | // work. :( The if statement below tests that condition and bails | ||||||||
| 2341 | // if it finds it. | ||||||||
| 2342 | bool DivIsSigned = Div->getOpcode() == Instruction::SDiv; | ||||||||
| 2343 | if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned()) | ||||||||
| 2344 | return nullptr; | ||||||||
| 2345 | |||||||||
| 2346 | // The ProdOV computation fails on divide by 0 and divide by -1. Cases with | ||||||||
| 2347 | // INT_MIN will also fail if the divisor is 1. Although folds of all these | ||||||||
| 2348 | // division-by-constant cases should be present, we can not assert that they | ||||||||
| 2349 | // have happened before we reach this icmp instruction. | ||||||||
| 2350 | if (C2->isNullValue() || C2->isOneValue() || | ||||||||
| 2351 | (DivIsSigned && C2->isAllOnesValue())) | ||||||||
| 2352 | return nullptr; | ||||||||
| 2353 | |||||||||
| 2354 | // Compute Prod = C * C2. We are essentially solving an equation of | ||||||||
| 2355 | // form X / C2 = C. We solve for X by multiplying C2 and C. | ||||||||
| 2356 | // By solving for X, we can turn this into a range check instead of computing | ||||||||
| 2357 | // a divide. | ||||||||
| 2358 | APInt Prod = C * *C2; | ||||||||
| 2359 | |||||||||
| 2360 | // Determine if the product overflows by seeing if the product is not equal to | ||||||||
| 2361 | // the divide. Make sure we do the same kind of divide as in the LHS | ||||||||
| 2362 | // instruction that we're folding. | ||||||||
| 2363 | bool ProdOV = (DivIsSigned ? Prod.sdiv(*C2) : Prod.udiv(*C2)) != C; | ||||||||
| 2364 | |||||||||
| 2365 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2366 | |||||||||
| 2367 | // If the division is known to be exact, then there is no remainder from the | ||||||||
| 2368 | // divide, so the covered range size is unit, otherwise it is the divisor. | ||||||||
| 2369 | APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2; | ||||||||
| 2370 | |||||||||
| 2371 | // Figure out the interval that is being checked. For example, a comparison | ||||||||
| 2372 | // like "X /u 5 == 0" is really checking that X is in the interval [0, 5). | ||||||||
| 2373 | // Compute this interval based on the constants involved and the signedness of | ||||||||
| 2374 | // the compare/divide. This computes a half-open interval, keeping track of | ||||||||
| 2375 | // whether either value in the interval overflows. After analysis each | ||||||||
| 2376 | // overflow variable is set to 0 if it's corresponding bound variable is valid | ||||||||
| 2377 | // -1 if overflowed off the bottom end, or +1 if overflowed off the top end. | ||||||||
| 2378 | int LoOverflow = 0, HiOverflow = 0; | ||||||||
| 2379 | APInt LoBound, HiBound; | ||||||||
| 2380 | |||||||||
| 2381 | if (!DivIsSigned) { // udiv | ||||||||
| 2382 | // e.g. X/5 op 3 --> [15, 20) | ||||||||
| 2383 | LoBound = Prod; | ||||||||
| 2384 | HiOverflow = LoOverflow = ProdOV; | ||||||||
| 2385 | if (!HiOverflow) { | ||||||||
| 2386 | // If this is not an exact divide, then many values in the range collapse | ||||||||
| 2387 | // to the same result value. | ||||||||
| 2388 | HiOverflow = addWithOverflow(HiBound, LoBound, RangeSize, false); | ||||||||
| 2389 | } | ||||||||
| 2390 | } else if (C2->isStrictlyPositive()) { // Divisor is > 0. | ||||||||
| 2391 | if (C.isNullValue()) { // (X / pos) op 0 | ||||||||
| 2392 | // Can't overflow. e.g. X/2 op 0 --> [-1, 2) | ||||||||
| 2393 | LoBound = -(RangeSize - 1); | ||||||||
| 2394 | HiBound = RangeSize; | ||||||||
| 2395 | } else if (C.isStrictlyPositive()) { // (X / pos) op pos | ||||||||
| 2396 | LoBound = Prod; // e.g. X/5 op 3 --> [15, 20) | ||||||||
| 2397 | HiOverflow = LoOverflow = ProdOV; | ||||||||
| 2398 | if (!HiOverflow) | ||||||||
| 2399 | HiOverflow = addWithOverflow(HiBound, Prod, RangeSize, true); | ||||||||
| 2400 | } else { // (X / pos) op neg | ||||||||
| 2401 | // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14) | ||||||||
| 2402 | HiBound = Prod + 1; | ||||||||
| 2403 | LoOverflow = HiOverflow = ProdOV ? -1 : 0; | ||||||||
| 2404 | if (!LoOverflow) { | ||||||||
| 2405 | APInt DivNeg = -RangeSize; | ||||||||
| 2406 | LoOverflow = addWithOverflow(LoBound, HiBound, DivNeg, true) ? -1 : 0; | ||||||||
| 2407 | } | ||||||||
| 2408 | } | ||||||||
| 2409 | } else if (C2->isNegative()) { // Divisor is < 0. | ||||||||
| 2410 | if (Div->isExact()) | ||||||||
| 2411 | RangeSize.negate(); | ||||||||
| 2412 | if (C.isNullValue()) { // (X / neg) op 0 | ||||||||
| 2413 | // e.g. X/-5 op 0 --> [-4, 5) | ||||||||
| 2414 | LoBound = RangeSize + 1; | ||||||||
| 2415 | HiBound = -RangeSize; | ||||||||
| 2416 | if (HiBound == *C2) { // -INTMIN = INTMIN | ||||||||
| 2417 | HiOverflow = 1; // [INTMIN+1, overflow) | ||||||||
| 2418 | HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN | ||||||||
| 2419 | } | ||||||||
| 2420 | } else if (C.isStrictlyPositive()) { // (X / neg) op pos | ||||||||
| 2421 | // e.g. X/-5 op 3 --> [-19, -14) | ||||||||
| 2422 | HiBound = Prod + 1; | ||||||||
| 2423 | HiOverflow = LoOverflow = ProdOV ? -1 : 0; | ||||||||
| 2424 | if (!LoOverflow) | ||||||||
| 2425 | LoOverflow = addWithOverflow(LoBound, HiBound, RangeSize, true) ? -1:0; | ||||||||
| 2426 | } else { // (X / neg) op neg | ||||||||
| 2427 | LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20) | ||||||||
| 2428 | LoOverflow = HiOverflow = ProdOV; | ||||||||
| 2429 | if (!HiOverflow) | ||||||||
| 2430 | HiOverflow = subWithOverflow(HiBound, Prod, RangeSize, true); | ||||||||
| 2431 | } | ||||||||
| 2432 | |||||||||
| 2433 | // Dividing by a negative swaps the condition. LT <-> GT | ||||||||
| 2434 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||||||
| 2435 | } | ||||||||
| 2436 | |||||||||
| 2437 | Value *X = Div->getOperand(0); | ||||||||
| 2438 | switch (Pred) { | ||||||||
| 2439 | default: llvm_unreachable("Unhandled icmp opcode!")::llvm::llvm_unreachable_internal("Unhandled icmp opcode!", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2439); | ||||||||
| 2440 | case ICmpInst::ICMP_EQ: | ||||||||
| 2441 | if (LoOverflow && HiOverflow) | ||||||||
| 2442 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
| 2443 | if (HiOverflow) | ||||||||
| 2444 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | ||||||||
| 2445 | ICmpInst::ICMP_UGE, X, | ||||||||
| 2446 | ConstantInt::get(Div->getType(), LoBound)); | ||||||||
| 2447 | if (LoOverflow) | ||||||||
| 2448 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | ||||||||
| 2449 | ICmpInst::ICMP_ULT, X, | ||||||||
| 2450 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
| 2451 | return replaceInstUsesWith( | ||||||||
| 2452 | Cmp, insertRangeTest(X, LoBound, HiBound, DivIsSigned, true)); | ||||||||
| 2453 | case ICmpInst::ICMP_NE: | ||||||||
| 2454 | if (LoOverflow && HiOverflow) | ||||||||
| 2455 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
| 2456 | if (HiOverflow) | ||||||||
| 2457 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : | ||||||||
| 2458 | ICmpInst::ICMP_ULT, X, | ||||||||
| 2459 | ConstantInt::get(Div->getType(), LoBound)); | ||||||||
| 2460 | if (LoOverflow) | ||||||||
| 2461 | return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : | ||||||||
| 2462 | ICmpInst::ICMP_UGE, X, | ||||||||
| 2463 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
| 2464 | return replaceInstUsesWith(Cmp, | ||||||||
| 2465 | insertRangeTest(X, LoBound, HiBound, | ||||||||
| 2466 | DivIsSigned, false)); | ||||||||
| 2467 | case ICmpInst::ICMP_ULT: | ||||||||
| 2468 | case ICmpInst::ICMP_SLT: | ||||||||
| 2469 | if (LoOverflow == +1) // Low bound is greater than input range. | ||||||||
| 2470 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
| 2471 | if (LoOverflow == -1) // Low bound is less than input range. | ||||||||
| 2472 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
| 2473 | return new ICmpInst(Pred, X, ConstantInt::get(Div->getType(), LoBound)); | ||||||||
| 2474 | case ICmpInst::ICMP_UGT: | ||||||||
| 2475 | case ICmpInst::ICMP_SGT: | ||||||||
| 2476 | if (HiOverflow == +1) // High bound greater than input range. | ||||||||
| 2477 | return replaceInstUsesWith(Cmp, Builder.getFalse()); | ||||||||
| 2478 | if (HiOverflow == -1) // High bound less than input range. | ||||||||
| 2479 | return replaceInstUsesWith(Cmp, Builder.getTrue()); | ||||||||
| 2480 | if (Pred == ICmpInst::ICMP_UGT) | ||||||||
| 2481 | return new ICmpInst(ICmpInst::ICMP_UGE, X, | ||||||||
| 2482 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
| 2483 | return new ICmpInst(ICmpInst::ICMP_SGE, X, | ||||||||
| 2484 | ConstantInt::get(Div->getType(), HiBound)); | ||||||||
| 2485 | } | ||||||||
| 2486 | |||||||||
| 2487 | return nullptr; | ||||||||
| 2488 | } | ||||||||
| 2489 | |||||||||
| 2490 | /// Fold icmp (sub X, Y), C. | ||||||||
| 2491 | Instruction *InstCombiner::foldICmpSubConstant(ICmpInst &Cmp, | ||||||||
| 2492 | BinaryOperator *Sub, | ||||||||
| 2493 | const APInt &C) { | ||||||||
| 2494 | Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1); | ||||||||
| 2495 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2496 | const APInt *C2; | ||||||||
| 2497 | APInt SubResult; | ||||||||
| 2498 | |||||||||
| 2499 | // icmp eq/ne (sub C, Y), C -> icmp eq/ne Y, 0 | ||||||||
| 2500 | if (match(X, m_APInt(C2)) && *C2 == C && Cmp.isEquality()) | ||||||||
| 2501 | return new ICmpInst(Cmp.getPredicate(), Y, | ||||||||
| 2502 | ConstantInt::get(Y->getType(), 0)); | ||||||||
| 2503 | |||||||||
| 2504 | // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C) | ||||||||
| 2505 | if (match(X, m_APInt(C2)) && | ||||||||
| 2506 | ((Cmp.isUnsigned() && Sub->hasNoUnsignedWrap()) || | ||||||||
| 2507 | (Cmp.isSigned() && Sub->hasNoSignedWrap())) && | ||||||||
| 2508 | !subWithOverflow(SubResult, *C2, C, Cmp.isSigned())) | ||||||||
| 2509 | return new ICmpInst(Cmp.getSwappedPredicate(), Y, | ||||||||
| 2510 | ConstantInt::get(Y->getType(), SubResult)); | ||||||||
| 2511 | |||||||||
| 2512 | // The following transforms are only worth it if the only user of the subtract | ||||||||
| 2513 | // is the icmp. | ||||||||
| 2514 | if (!Sub->hasOneUse()) | ||||||||
| 2515 | return nullptr; | ||||||||
| 2516 | |||||||||
| 2517 | if (Sub->hasNoSignedWrap()) { | ||||||||
| 2518 | // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y) | ||||||||
| 2519 | if (Pred == ICmpInst::ICMP_SGT && C.isAllOnesValue()) | ||||||||
| 2520 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | ||||||||
| 2521 | |||||||||
| 2522 | // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y) | ||||||||
| 2523 | if (Pred == ICmpInst::ICMP_SGT && C.isNullValue()) | ||||||||
| 2524 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | ||||||||
| 2525 | |||||||||
| 2526 | // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y) | ||||||||
| 2527 | if (Pred == ICmpInst::ICMP_SLT && C.isNullValue()) | ||||||||
| 2528 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | ||||||||
| 2529 | |||||||||
| 2530 | // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y) | ||||||||
| 2531 | if (Pred == ICmpInst::ICMP_SLT && C.isOneValue()) | ||||||||
| 2532 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | ||||||||
| 2533 | } | ||||||||
| 2534 | |||||||||
| 2535 | if (!match(X, m_APInt(C2))) | ||||||||
| 2536 | return nullptr; | ||||||||
| 2537 | |||||||||
| 2538 | // C2 - Y <u C -> (Y | (C - 1)) == C2 | ||||||||
| 2539 | // iff (C2 & (C - 1)) == C - 1 and C is a power of 2 | ||||||||
| 2540 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && | ||||||||
| 2541 | (*C2 & (C - 1)) == (C - 1)) | ||||||||
| 2542 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(Y, C - 1), X); | ||||||||
| 2543 | |||||||||
| 2544 | // C2 - Y >u C -> (Y | C) != C2 | ||||||||
| 2545 | // iff C2 & C == C and C + 1 is a power of 2 | ||||||||
| 2546 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C) | ||||||||
| 2547 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(Y, C), X); | ||||||||
| 2548 | |||||||||
| 2549 | return nullptr; | ||||||||
| 2550 | } | ||||||||
| 2551 | |||||||||
| 2552 | /// Fold icmp (add X, Y), C. | ||||||||
| 2553 | Instruction *InstCombiner::foldICmpAddConstant(ICmpInst &Cmp, | ||||||||
| 2554 | BinaryOperator *Add, | ||||||||
| 2555 | const APInt &C) { | ||||||||
| 2556 | Value *Y = Add->getOperand(1); | ||||||||
| 2557 | const APInt *C2; | ||||||||
| 2558 | if (Cmp.isEquality() || !match(Y, m_APInt(C2))) | ||||||||
| 2559 | return nullptr; | ||||||||
| 2560 | |||||||||
| 2561 | // Fold icmp pred (add X, C2), C. | ||||||||
| 2562 | Value *X = Add->getOperand(0); | ||||||||
| 2563 | Type *Ty = Add->getType(); | ||||||||
| 2564 | CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2565 | |||||||||
| 2566 | // If the add does not wrap, we can always adjust the compare by subtracting | ||||||||
| 2567 | // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE | ||||||||
| 2568 | // are canonicalized to SGT/SLT/UGT/ULT. | ||||||||
| 2569 | if ((Add->hasNoSignedWrap() && | ||||||||
| 2570 | (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) || | ||||||||
| 2571 | (Add->hasNoUnsignedWrap() && | ||||||||
| 2572 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) { | ||||||||
| 2573 | bool Overflow; | ||||||||
| 2574 | APInt NewC = | ||||||||
| 2575 | Cmp.isSigned() ? C.ssub_ov(*C2, Overflow) : C.usub_ov(*C2, Overflow); | ||||||||
| 2576 | // If there is overflow, the result must be true or false. | ||||||||
| 2577 | // TODO: Can we assert there is no overflow because InstSimplify always | ||||||||
| 2578 | // handles those cases? | ||||||||
| 2579 | if (!Overflow) | ||||||||
| 2580 | // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2) | ||||||||
| 2581 | return new ICmpInst(Pred, X, ConstantInt::get(Ty, NewC)); | ||||||||
| 2582 | } | ||||||||
| 2583 | |||||||||
| 2584 | auto CR = ConstantRange::makeExactICmpRegion(Pred, C).subtract(*C2); | ||||||||
| 2585 | const APInt &Upper = CR.getUpper(); | ||||||||
| 2586 | const APInt &Lower = CR.getLower(); | ||||||||
| 2587 | if (Cmp.isSigned()) { | ||||||||
| 2588 | if (Lower.isSignMask()) | ||||||||
| 2589 | return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, Upper)); | ||||||||
| 2590 | if (Upper.isSignMask()) | ||||||||
| 2591 | return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, Lower)); | ||||||||
| 2592 | } else { | ||||||||
| 2593 | if (Lower.isMinValue()) | ||||||||
| 2594 | return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, Upper)); | ||||||||
| 2595 | if (Upper.isMinValue()) | ||||||||
| 2596 | return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, Lower)); | ||||||||
| 2597 | } | ||||||||
| 2598 | |||||||||
| 2599 | if (!Add->hasOneUse()) | ||||||||
| 2600 | return nullptr; | ||||||||
| 2601 | |||||||||
| 2602 | // X+C <u C2 -> (X & -C2) == C | ||||||||
| 2603 | // iff C & (C2-1) == 0 | ||||||||
| 2604 | // C2 is a power of 2 | ||||||||
| 2605 | if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0) | ||||||||
| 2606 | return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(X, -C), | ||||||||
| 2607 | ConstantExpr::getNeg(cast<Constant>(Y))); | ||||||||
| 2608 | |||||||||
| 2609 | // X+C >u C2 -> (X & ~C2) != C | ||||||||
| 2610 | // iff C & C2 == 0 | ||||||||
| 2611 | // C2+1 is a power of 2 | ||||||||
| 2612 | if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0) | ||||||||
| 2613 | return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(X, ~C), | ||||||||
| 2614 | ConstantExpr::getNeg(cast<Constant>(Y))); | ||||||||
| 2615 | |||||||||
| 2616 | return nullptr; | ||||||||
| 2617 | } | ||||||||
| 2618 | |||||||||
| 2619 | bool InstCombiner::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS, | ||||||||
| 2620 | Value *&RHS, ConstantInt *&Less, | ||||||||
| 2621 | ConstantInt *&Equal, | ||||||||
| 2622 | ConstantInt *&Greater) { | ||||||||
| 2623 | // TODO: Generalize this to work with other comparison idioms or ensure | ||||||||
| 2624 | // they get canonicalized into this form. | ||||||||
| 2625 | |||||||||
| 2626 | // select i1 (a == b), | ||||||||
| 2627 | // i32 Equal, | ||||||||
| 2628 | // i32 (select i1 (a < b), i32 Less, i32 Greater) | ||||||||
| 2629 | // where Equal, Less and Greater are placeholders for any three constants. | ||||||||
| 2630 | ICmpInst::Predicate PredA; | ||||||||
| 2631 | if (!match(SI->getCondition(), m_ICmp(PredA, m_Value(LHS), m_Value(RHS))) || | ||||||||
| 2632 | !ICmpInst::isEquality(PredA)) | ||||||||
| 2633 | return false; | ||||||||
| 2634 | Value *EqualVal = SI->getTrueValue(); | ||||||||
| 2635 | Value *UnequalVal = SI->getFalseValue(); | ||||||||
| 2636 | // We still can get non-canonical predicate here, so canonicalize. | ||||||||
| 2637 | if (PredA == ICmpInst::ICMP_NE) | ||||||||
| 2638 | std::swap(EqualVal, UnequalVal); | ||||||||
| 2639 | if (!match(EqualVal, m_ConstantInt(Equal))) | ||||||||
| 2640 | return false; | ||||||||
| 2641 | ICmpInst::Predicate PredB; | ||||||||
| 2642 | Value *LHS2, *RHS2; | ||||||||
| 2643 | if (!match(UnequalVal, m_Select(m_ICmp(PredB, m_Value(LHS2), m_Value(RHS2)), | ||||||||
| 2644 | m_ConstantInt(Less), m_ConstantInt(Greater)))) | ||||||||
| 2645 | return false; | ||||||||
| 2646 | // We can get predicate mismatch here, so canonicalize if possible: | ||||||||
| 2647 | // First, ensure that 'LHS' match. | ||||||||
| 2648 | if (LHS2 != LHS) { | ||||||||
| 2649 | // x sgt y <--> y slt x | ||||||||
| 2650 | std::swap(LHS2, RHS2); | ||||||||
| 2651 | PredB = ICmpInst::getSwappedPredicate(PredB); | ||||||||
| 2652 | } | ||||||||
| 2653 | if (LHS2 != LHS) | ||||||||
| 2654 | return false; | ||||||||
| 2655 | // We also need to canonicalize 'RHS'. | ||||||||
| 2656 | if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) { | ||||||||
| 2657 | // x sgt C-1 <--> x sge C <--> not(x slt C) | ||||||||
| 2658 | auto FlippedStrictness = | ||||||||
| 2659 | getFlippedStrictnessPredicateAndConstant(PredB, cast<Constant>(RHS2)); | ||||||||
| 2660 | if (!FlippedStrictness) | ||||||||
| 2661 | return false; | ||||||||
| 2662 | assert(FlippedStrictness->first == ICmpInst::ICMP_SGE && "Sanity check")((FlippedStrictness->first == ICmpInst::ICMP_SGE && "Sanity check") ? static_cast<void> (0) : __assert_fail ("FlippedStrictness->first == ICmpInst::ICMP_SGE && \"Sanity check\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2662, __PRETTY_FUNCTION__)); | ||||||||
| 2663 | RHS2 = FlippedStrictness->second; | ||||||||
| 2664 | // And kind-of perform the result swap. | ||||||||
| 2665 | std::swap(Less, Greater); | ||||||||
| 2666 | PredB = ICmpInst::ICMP_SLT; | ||||||||
| 2667 | } | ||||||||
| 2668 | return PredB == ICmpInst::ICMP_SLT && RHS == RHS2; | ||||||||
| 2669 | } | ||||||||
| 2670 | |||||||||
| 2671 | Instruction *InstCombiner::foldICmpSelectConstant(ICmpInst &Cmp, | ||||||||
| 2672 | SelectInst *Select, | ||||||||
| 2673 | ConstantInt *C) { | ||||||||
| 2674 | |||||||||
| 2675 | assert(C && "Cmp RHS should be a constant int!")((C && "Cmp RHS should be a constant int!") ? static_cast <void> (0) : __assert_fail ("C && \"Cmp RHS should be a constant int!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2675, __PRETTY_FUNCTION__)); | ||||||||
| 2676 | // If we're testing a constant value against the result of a three way | ||||||||
| 2677 | // comparison, the result can be expressed directly in terms of the | ||||||||
| 2678 | // original values being compared. Note: We could possibly be more | ||||||||
| 2679 | // aggressive here and remove the hasOneUse test. The original select is | ||||||||
| 2680 | // really likely to simplify or sink when we remove a test of the result. | ||||||||
| 2681 | Value *OrigLHS, *OrigRHS; | ||||||||
| 2682 | ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan; | ||||||||
| 2683 | if (Cmp.hasOneUse() && | ||||||||
| 2684 | matchThreeWayIntCompare(Select, OrigLHS, OrigRHS, C1LessThan, C2Equal, | ||||||||
| 2685 | C3GreaterThan)) { | ||||||||
| 2686 | assert(C1LessThan && C2Equal && C3GreaterThan)((C1LessThan && C2Equal && C3GreaterThan) ? static_cast <void> (0) : __assert_fail ("C1LessThan && C2Equal && C3GreaterThan" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 2686, __PRETTY_FUNCTION__)); | ||||||||
| 2687 | |||||||||
| 2688 | bool TrueWhenLessThan = | ||||||||
| 2689 | ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C) | ||||||||
| 2690 | ->isAllOnesValue(); | ||||||||
| 2691 | bool TrueWhenEqual = | ||||||||
| 2692 | ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C) | ||||||||
| 2693 | ->isAllOnesValue(); | ||||||||
| 2694 | bool TrueWhenGreaterThan = | ||||||||
| 2695 | ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C) | ||||||||
| 2696 | ->isAllOnesValue(); | ||||||||
| 2697 | |||||||||
| 2698 | // This generates the new instruction that will replace the original Cmp | ||||||||
| 2699 | // Instruction. Instead of enumerating the various combinations when | ||||||||
| 2700 | // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus | ||||||||
| 2701 | // false, we rely on chaining of ORs and future passes of InstCombine to | ||||||||
| 2702 | // simplify the OR further (i.e. a s< b || a == b becomes a s<= b). | ||||||||
| 2703 | |||||||||
| 2704 | // When none of the three constants satisfy the predicate for the RHS (C), | ||||||||
| 2705 | // the entire original Cmp can be simplified to a false. | ||||||||
| 2706 | Value *Cond = Builder.getFalse(); | ||||||||
| 2707 | if (TrueWhenLessThan) | ||||||||
| 2708 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, | ||||||||
| 2709 | OrigLHS, OrigRHS)); | ||||||||
| 2710 | if (TrueWhenEqual) | ||||||||
| 2711 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, | ||||||||
| 2712 | OrigLHS, OrigRHS)); | ||||||||
| 2713 | if (TrueWhenGreaterThan) | ||||||||
| 2714 | Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, | ||||||||
| 2715 | OrigLHS, OrigRHS)); | ||||||||
| 2716 | |||||||||
| 2717 | return replaceInstUsesWith(Cmp, Cond); | ||||||||
| 2718 | } | ||||||||
| 2719 | return nullptr; | ||||||||
| 2720 | } | ||||||||
| 2721 | |||||||||
| 2722 | static Instruction *foldICmpBitCast(ICmpInst &Cmp, | ||||||||
| 2723 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 2724 | auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0)); | ||||||||
| 2725 | if (!Bitcast) | ||||||||
| 2726 | return nullptr; | ||||||||
| 2727 | |||||||||
| 2728 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2729 | Value *Op1 = Cmp.getOperand(1); | ||||||||
| 2730 | Value *BCSrcOp = Bitcast->getOperand(0); | ||||||||
| 2731 | |||||||||
| 2732 | // Make sure the bitcast doesn't change the number of vector elements. | ||||||||
| 2733 | if (Bitcast->getSrcTy()->getScalarSizeInBits() == | ||||||||
| 2734 | Bitcast->getDestTy()->getScalarSizeInBits()) { | ||||||||
| 2735 | // Zero-equality and sign-bit checks are preserved through sitofp + bitcast. | ||||||||
| 2736 | Value *X; | ||||||||
| 2737 | if (match(BCSrcOp, m_SIToFP(m_Value(X)))) { | ||||||||
| 2738 | // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0 | ||||||||
| 2739 | // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0 | ||||||||
| 2740 | // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0 | ||||||||
| 2741 | // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0 | ||||||||
| 2742 | if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT || | ||||||||
| 2743 | Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) && | ||||||||
| 2744 | match(Op1, m_Zero())) | ||||||||
| 2745 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); | ||||||||
| 2746 | |||||||||
| 2747 | // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1 | ||||||||
| 2748 | if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One())) | ||||||||
| 2749 | return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1)); | ||||||||
| 2750 | |||||||||
| 2751 | // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1 | ||||||||
| 2752 | if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes())) | ||||||||
| 2753 | return new ICmpInst(Pred, X, | ||||||||
| 2754 | ConstantInt::getAllOnesValue(X->getType())); | ||||||||
| 2755 | } | ||||||||
| 2756 | |||||||||
| 2757 | // Zero-equality checks are preserved through unsigned floating-point casts: | ||||||||
| 2758 | // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0 | ||||||||
| 2759 | // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0 | ||||||||
| 2760 | if (match(BCSrcOp, m_UIToFP(m_Value(X)))) | ||||||||
| 2761 | if (Cmp.isEquality() && match(Op1, m_Zero())) | ||||||||
| 2762 | return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType())); | ||||||||
| 2763 | } | ||||||||
| 2764 | |||||||||
| 2765 | // Test to see if the operands of the icmp are casted versions of other | ||||||||
| 2766 | // values. If the ptr->ptr cast can be stripped off both arguments, do so. | ||||||||
| 2767 | if (Bitcast->getType()->isPointerTy() && | ||||||||
| 2768 | (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) { | ||||||||
| 2769 | // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast | ||||||||
| 2770 | // so eliminate it as well. | ||||||||
| 2771 | if (auto *BC2 = dyn_cast<BitCastInst>(Op1)) | ||||||||
| 2772 | Op1 = BC2->getOperand(0); | ||||||||
| 2773 | |||||||||
| 2774 | Op1 = Builder.CreateBitCast(Op1, BCSrcOp->getType()); | ||||||||
| 2775 | return new ICmpInst(Pred, BCSrcOp, Op1); | ||||||||
| 2776 | } | ||||||||
| 2777 | |||||||||
| 2778 | // Folding: icmp <pred> iN X, C | ||||||||
| 2779 | // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN | ||||||||
| 2780 | // and C is a splat of a K-bit pattern | ||||||||
| 2781 | // and SC is a constant vector = <C', C', C', ..., C'> | ||||||||
| 2782 | // Into: | ||||||||
| 2783 | // %E = extractelement <M x iK> %vec, i32 C' | ||||||||
| 2784 | // icmp <pred> iK %E, trunc(C) | ||||||||
| 2785 | const APInt *C; | ||||||||
| 2786 | if (!match(Cmp.getOperand(1), m_APInt(C)) || | ||||||||
| 2787 | !Bitcast->getType()->isIntegerTy() || | ||||||||
| 2788 | !Bitcast->getSrcTy()->isIntOrIntVectorTy()) | ||||||||
| 2789 | return nullptr; | ||||||||
| 2790 | |||||||||
| 2791 | Value *Vec; | ||||||||
| 2792 | Constant *Mask; | ||||||||
| 2793 | if (match(BCSrcOp, | ||||||||
| 2794 | m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) { | ||||||||
| 2795 | // Check whether every element of Mask is the same constant | ||||||||
| 2796 | if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) { | ||||||||
| 2797 | auto *VecTy = cast<VectorType>(BCSrcOp->getType()); | ||||||||
| 2798 | auto *EltTy = cast<IntegerType>(VecTy->getElementType()); | ||||||||
| 2799 | if (C->isSplat(EltTy->getBitWidth())) { | ||||||||
| 2800 | // Fold the icmp based on the value of C | ||||||||
| 2801 | // If C is M copies of an iK sized bit pattern, | ||||||||
| 2802 | // then: | ||||||||
| 2803 | // => %E = extractelement <N x iK> %vec, i32 Elem | ||||||||
| 2804 | // icmp <pred> iK %SplatVal, <pattern> | ||||||||
| 2805 | Value *Extract = Builder.CreateExtractElement(Vec, Elem); | ||||||||
| 2806 | Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth())); | ||||||||
| 2807 | return new ICmpInst(Pred, Extract, NewC); | ||||||||
| 2808 | } | ||||||||
| 2809 | } | ||||||||
| 2810 | } | ||||||||
| 2811 | return nullptr; | ||||||||
| 2812 | } | ||||||||
| 2813 | |||||||||
| 2814 | /// Try to fold integer comparisons with a constant operand: icmp Pred X, C | ||||||||
| 2815 | /// where X is some kind of instruction. | ||||||||
| 2816 | Instruction *InstCombiner::foldICmpInstWithConstant(ICmpInst &Cmp) { | ||||||||
| 2817 | const APInt *C; | ||||||||
| 2818 | if (!match(Cmp.getOperand(1), m_APInt(C))) | ||||||||
| 2819 | return nullptr; | ||||||||
| 2820 | |||||||||
| 2821 | if (auto *BO = dyn_cast<BinaryOperator>(Cmp.getOperand(0))) { | ||||||||
| 2822 | switch (BO->getOpcode()) { | ||||||||
| 2823 | case Instruction::Xor: | ||||||||
| 2824 | if (Instruction *I = foldICmpXorConstant(Cmp, BO, *C)) | ||||||||
| 2825 | return I; | ||||||||
| 2826 | break; | ||||||||
| 2827 | case Instruction::And: | ||||||||
| 2828 | if (Instruction *I = foldICmpAndConstant(Cmp, BO, *C)) | ||||||||
| 2829 | return I; | ||||||||
| 2830 | break; | ||||||||
| 2831 | case Instruction::Or: | ||||||||
| 2832 | if (Instruction *I = foldICmpOrConstant(Cmp, BO, *C)) | ||||||||
| 2833 | return I; | ||||||||
| 2834 | break; | ||||||||
| 2835 | case Instruction::Mul: | ||||||||
| 2836 | if (Instruction *I = foldICmpMulConstant(Cmp, BO, *C)) | ||||||||
| 2837 | return I; | ||||||||
| 2838 | break; | ||||||||
| 2839 | case Instruction::Shl: | ||||||||
| 2840 | if (Instruction *I = foldICmpShlConstant(Cmp, BO, *C)) | ||||||||
| 2841 | return I; | ||||||||
| 2842 | break; | ||||||||
| 2843 | case Instruction::LShr: | ||||||||
| 2844 | case Instruction::AShr: | ||||||||
| 2845 | if (Instruction *I = foldICmpShrConstant(Cmp, BO, *C)) | ||||||||
| 2846 | return I; | ||||||||
| 2847 | break; | ||||||||
| 2848 | case Instruction::SRem: | ||||||||
| 2849 | if (Instruction *I = foldICmpSRemConstant(Cmp, BO, *C)) | ||||||||
| 2850 | return I; | ||||||||
| 2851 | break; | ||||||||
| 2852 | case Instruction::UDiv: | ||||||||
| 2853 | if (Instruction *I = foldICmpUDivConstant(Cmp, BO, *C)) | ||||||||
| 2854 | return I; | ||||||||
| 2855 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
| 2856 | case Instruction::SDiv: | ||||||||
| 2857 | if (Instruction *I = foldICmpDivConstant(Cmp, BO, *C)) | ||||||||
| 2858 | return I; | ||||||||
| 2859 | break; | ||||||||
| 2860 | case Instruction::Sub: | ||||||||
| 2861 | if (Instruction *I = foldICmpSubConstant(Cmp, BO, *C)) | ||||||||
| 2862 | return I; | ||||||||
| 2863 | break; | ||||||||
| 2864 | case Instruction::Add: | ||||||||
| 2865 | if (Instruction *I = foldICmpAddConstant(Cmp, BO, *C)) | ||||||||
| 2866 | return I; | ||||||||
| 2867 | break; | ||||||||
| 2868 | default: | ||||||||
| 2869 | break; | ||||||||
| 2870 | } | ||||||||
| 2871 | // TODO: These folds could be refactored to be part of the above calls. | ||||||||
| 2872 | if (Instruction *I = foldICmpBinOpEqualityWithConstant(Cmp, BO, *C)) | ||||||||
| 2873 | return I; | ||||||||
| 2874 | } | ||||||||
| 2875 | |||||||||
| 2876 | // Match against CmpInst LHS being instructions other than binary operators. | ||||||||
| 2877 | |||||||||
| 2878 | if (auto *SI = dyn_cast<SelectInst>(Cmp.getOperand(0))) { | ||||||||
| 2879 | // For now, we only support constant integers while folding the | ||||||||
| 2880 | // ICMP(SELECT)) pattern. We can extend this to support vector of integers | ||||||||
| 2881 | // similar to the cases handled by binary ops above. | ||||||||
| 2882 | if (ConstantInt *ConstRHS = dyn_cast<ConstantInt>(Cmp.getOperand(1))) | ||||||||
| 2883 | if (Instruction *I = foldICmpSelectConstant(Cmp, SI, ConstRHS)) | ||||||||
| 2884 | return I; | ||||||||
| 2885 | } | ||||||||
| 2886 | |||||||||
| 2887 | if (auto *TI = dyn_cast<TruncInst>(Cmp.getOperand(0))) { | ||||||||
| 2888 | if (Instruction *I = foldICmpTruncConstant(Cmp, TI, *C)) | ||||||||
| 2889 | return I; | ||||||||
| 2890 | } | ||||||||
| 2891 | |||||||||
| 2892 | if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0))) | ||||||||
| 2893 | if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C)) | ||||||||
| 2894 | return I; | ||||||||
| 2895 | |||||||||
| 2896 | return nullptr; | ||||||||
| 2897 | } | ||||||||
| 2898 | |||||||||
| 2899 | /// Fold an icmp equality instruction with binary operator LHS and constant RHS: | ||||||||
| 2900 | /// icmp eq/ne BO, C. | ||||||||
| 2901 | Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp, | ||||||||
| 2902 | BinaryOperator *BO, | ||||||||
| 2903 | const APInt &C) { | ||||||||
| 2904 | // TODO: Some of these folds could work with arbitrary constants, but this | ||||||||
| 2905 | // function is limited to scalar and vector splat constants. | ||||||||
| 2906 | if (!Cmp.isEquality()) | ||||||||
| 2907 | return nullptr; | ||||||||
| 2908 | |||||||||
| 2909 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 2910 | bool isICMP_NE = Pred == ICmpInst::ICMP_NE; | ||||||||
| 2911 | Constant *RHS = cast<Constant>(Cmp.getOperand(1)); | ||||||||
| 2912 | Value *BOp0 = BO->getOperand(0), *BOp1 = BO->getOperand(1); | ||||||||
| 2913 | |||||||||
| 2914 | switch (BO->getOpcode()) { | ||||||||
| 2915 | case Instruction::SRem: | ||||||||
| 2916 | // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one. | ||||||||
| 2917 | if (C.isNullValue() && BO->hasOneUse()) { | ||||||||
| 2918 | const APInt *BOC; | ||||||||
| 2919 | if (match(BOp1, m_APInt(BOC)) && BOC->sgt(1) && BOC->isPowerOf2()) { | ||||||||
| 2920 | Value *NewRem = Builder.CreateURem(BOp0, BOp1, BO->getName()); | ||||||||
| 2921 | return new ICmpInst(Pred, NewRem, | ||||||||
| 2922 | Constant::getNullValue(BO->getType())); | ||||||||
| 2923 | } | ||||||||
| 2924 | } | ||||||||
| 2925 | break; | ||||||||
| 2926 | case Instruction::Add: { | ||||||||
| 2927 | // Replace ((add A, B) != C) with (A != C-B) if B & C are constants. | ||||||||
| 2928 | if (Constant *BOC = dyn_cast<Constant>(BOp1)) { | ||||||||
| 2929 | if (BO->hasOneUse()) | ||||||||
| 2930 | return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(RHS, BOC)); | ||||||||
| 2931 | } else if (C.isNullValue()) { | ||||||||
| 2932 | // Replace ((add A, B) != 0) with (A != -B) if A or B is | ||||||||
| 2933 | // efficiently invertible, or if the add has just this one use. | ||||||||
| 2934 | if (Value *NegVal = dyn_castNegVal(BOp1)) | ||||||||
| 2935 | return new ICmpInst(Pred, BOp0, NegVal); | ||||||||
| 2936 | if (Value *NegVal = dyn_castNegVal(BOp0)) | ||||||||
| 2937 | return new ICmpInst(Pred, NegVal, BOp1); | ||||||||
| 2938 | if (BO->hasOneUse()) { | ||||||||
| 2939 | Value *Neg = Builder.CreateNeg(BOp1); | ||||||||
| 2940 | Neg->takeName(BO); | ||||||||
| 2941 | return new ICmpInst(Pred, BOp0, Neg); | ||||||||
| 2942 | } | ||||||||
| 2943 | } | ||||||||
| 2944 | break; | ||||||||
| 2945 | } | ||||||||
| 2946 | case Instruction::Xor: | ||||||||
| 2947 | if (BO->hasOneUse()) { | ||||||||
| 2948 | if (Constant *BOC = dyn_cast<Constant>(BOp1)) { | ||||||||
| 2949 | // For the xor case, we can xor two constants together, eliminating | ||||||||
| 2950 | // the explicit xor. | ||||||||
| 2951 | return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(RHS, BOC)); | ||||||||
| 2952 | } else if (C.isNullValue()) { | ||||||||
| 2953 | // Replace ((xor A, B) != 0) with (A != B) | ||||||||
| 2954 | return new ICmpInst(Pred, BOp0, BOp1); | ||||||||
| 2955 | } | ||||||||
| 2956 | } | ||||||||
| 2957 | break; | ||||||||
| 2958 | case Instruction::Sub: | ||||||||
| 2959 | if (BO->hasOneUse()) { | ||||||||
| 2960 | // Only check for constant LHS here, as constant RHS will be canonicalized | ||||||||
| 2961 | // to add and use the fold above. | ||||||||
| 2962 | if (Constant *BOC = dyn_cast<Constant>(BOp0)) { | ||||||||
| 2963 | // Replace ((sub BOC, B) != C) with (B != BOC-C). | ||||||||
| 2964 | return new ICmpInst(Pred, BOp1, ConstantExpr::getSub(BOC, RHS)); | ||||||||
| 2965 | } else if (C.isNullValue()) { | ||||||||
| 2966 | // Replace ((sub A, B) != 0) with (A != B). | ||||||||
| 2967 | return new ICmpInst(Pred, BOp0, BOp1); | ||||||||
| 2968 | } | ||||||||
| 2969 | } | ||||||||
| 2970 | break; | ||||||||
| 2971 | case Instruction::Or: { | ||||||||
| 2972 | const APInt *BOC; | ||||||||
| 2973 | if (match(BOp1, m_APInt(BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) { | ||||||||
| 2974 | // Comparing if all bits outside of a constant mask are set? | ||||||||
| 2975 | // Replace (X | C) == -1 with (X & ~C) == ~C. | ||||||||
| 2976 | // This removes the -1 constant. | ||||||||
| 2977 | Constant *NotBOC = ConstantExpr::getNot(cast<Constant>(BOp1)); | ||||||||
| 2978 | Value *And = Builder.CreateAnd(BOp0, NotBOC); | ||||||||
| 2979 | return new ICmpInst(Pred, And, NotBOC); | ||||||||
| 2980 | } | ||||||||
| 2981 | break; | ||||||||
| 2982 | } | ||||||||
| 2983 | case Instruction::And: { | ||||||||
| 2984 | const APInt *BOC; | ||||||||
| 2985 | if (match(BOp1, m_APInt(BOC))) { | ||||||||
| 2986 | // If we have ((X & C) == C), turn it into ((X & C) != 0). | ||||||||
| 2987 | if (C == *BOC && C.isPowerOf2()) | ||||||||
| 2988 | return new ICmpInst(isICMP_NE ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE, | ||||||||
| 2989 | BO, Constant::getNullValue(RHS->getType())); | ||||||||
| 2990 | } | ||||||||
| 2991 | break; | ||||||||
| 2992 | } | ||||||||
| 2993 | case Instruction::Mul: | ||||||||
| 2994 | if (C.isNullValue() && BO->hasNoSignedWrap()) { | ||||||||
| 2995 | const APInt *BOC; | ||||||||
| 2996 | if (match(BOp1, m_APInt(BOC)) && !BOC->isNullValue()) { | ||||||||
| 2997 | // The trivial case (mul X, 0) is handled by InstSimplify. | ||||||||
| 2998 | // General case : (mul X, C) != 0 iff X != 0 | ||||||||
| 2999 | // (mul X, C) == 0 iff X == 0 | ||||||||
| 3000 | return new ICmpInst(Pred, BOp0, Constant::getNullValue(RHS->getType())); | ||||||||
| 3001 | } | ||||||||
| 3002 | } | ||||||||
| 3003 | break; | ||||||||
| 3004 | case Instruction::UDiv: | ||||||||
| 3005 | if (C.isNullValue()) { | ||||||||
| 3006 | // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A) | ||||||||
| 3007 | auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; | ||||||||
| 3008 | return new ICmpInst(NewPred, BOp1, BOp0); | ||||||||
| 3009 | } | ||||||||
| 3010 | break; | ||||||||
| 3011 | default: | ||||||||
| 3012 | break; | ||||||||
| 3013 | } | ||||||||
| 3014 | return nullptr; | ||||||||
| 3015 | } | ||||||||
| 3016 | |||||||||
| 3017 | /// Fold an equality icmp with LLVM intrinsic and constant operand. | ||||||||
| 3018 | Instruction *InstCombiner::foldICmpEqIntrinsicWithConstant(ICmpInst &Cmp, | ||||||||
| 3019 | IntrinsicInst *II, | ||||||||
| 3020 | const APInt &C) { | ||||||||
| 3021 | Type *Ty = II->getType(); | ||||||||
| 3022 | unsigned BitWidth = C.getBitWidth(); | ||||||||
| 3023 | switch (II->getIntrinsicID()) { | ||||||||
| 3024 | case Intrinsic::bswap: | ||||||||
| 3025 | // bswap(A) == C -> A == bswap(C) | ||||||||
| 3026 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||||||
| 3027 | ConstantInt::get(Ty, C.byteSwap())); | ||||||||
| 3028 | |||||||||
| 3029 | case Intrinsic::ctlz: | ||||||||
| 3030 | case Intrinsic::cttz: { | ||||||||
| 3031 | // ctz(A) == bitwidth(A) -> A == 0 and likewise for != | ||||||||
| 3032 | if (C == BitWidth) | ||||||||
| 3033 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||||||
| 3034 | ConstantInt::getNullValue(Ty)); | ||||||||
| 3035 | |||||||||
| 3036 | // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set | ||||||||
| 3037 | // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits. | ||||||||
| 3038 | // Limit to one use to ensure we don't increase instruction count. | ||||||||
| 3039 | unsigned Num = C.getLimitedValue(BitWidth); | ||||||||
| 3040 | if (Num != BitWidth && II->hasOneUse()) { | ||||||||
| 3041 | bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz; | ||||||||
| 3042 | APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(BitWidth, Num + 1) | ||||||||
| 3043 | : APInt::getHighBitsSet(BitWidth, Num + 1); | ||||||||
| 3044 | APInt Mask2 = IsTrailing | ||||||||
| 3045 | ? APInt::getOneBitSet(BitWidth, Num) | ||||||||
| 3046 | : APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); | ||||||||
| 3047 | return new ICmpInst(Cmp.getPredicate(), | ||||||||
| 3048 | Builder.CreateAnd(II->getArgOperand(0), Mask1), | ||||||||
| 3049 | ConstantInt::get(Ty, Mask2)); | ||||||||
| 3050 | } | ||||||||
| 3051 | break; | ||||||||
| 3052 | } | ||||||||
| 3053 | |||||||||
| 3054 | case Intrinsic::ctpop: { | ||||||||
| 3055 | // popcount(A) == 0 -> A == 0 and likewise for != | ||||||||
| 3056 | // popcount(A) == bitwidth(A) -> A == -1 and likewise for != | ||||||||
| 3057 | bool IsZero = C.isNullValue(); | ||||||||
| 3058 | if (IsZero || C == BitWidth) | ||||||||
| 3059 | return new ICmpInst(Cmp.getPredicate(), II->getArgOperand(0), | ||||||||
| 3060 | IsZero ? Constant::getNullValue(Ty) : Constant::getAllOnesValue(Ty)); | ||||||||
| 3061 | |||||||||
| 3062 | break; | ||||||||
| 3063 | } | ||||||||
| 3064 | |||||||||
| 3065 | case Intrinsic::uadd_sat: { | ||||||||
| 3066 | // uadd.sat(a, b) == 0 -> (a | b) == 0 | ||||||||
| 3067 | if (C.isNullValue()) { | ||||||||
| 3068 | Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1)); | ||||||||
| 3069 | return new ICmpInst(Cmp.getPredicate(), Or, Constant::getNullValue(Ty)); | ||||||||
| 3070 | } | ||||||||
| 3071 | break; | ||||||||
| 3072 | } | ||||||||
| 3073 | |||||||||
| 3074 | case Intrinsic::usub_sat: { | ||||||||
| 3075 | // usub.sat(a, b) == 0 -> a <= b | ||||||||
| 3076 | if (C.isNullValue()) { | ||||||||
| 3077 | ICmpInst::Predicate NewPred = Cmp.getPredicate() == ICmpInst::ICMP_EQ | ||||||||
| 3078 | ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT; | ||||||||
| 3079 | return new ICmpInst(NewPred, II->getArgOperand(0), II->getArgOperand(1)); | ||||||||
| 3080 | } | ||||||||
| 3081 | break; | ||||||||
| 3082 | } | ||||||||
| 3083 | default: | ||||||||
| 3084 | break; | ||||||||
| 3085 | } | ||||||||
| 3086 | |||||||||
| 3087 | return nullptr; | ||||||||
| 3088 | } | ||||||||
| 3089 | |||||||||
| 3090 | /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C. | ||||||||
| 3091 | Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp, | ||||||||
| 3092 | IntrinsicInst *II, | ||||||||
| 3093 | const APInt &C) { | ||||||||
| 3094 | if (Cmp.isEquality()) | ||||||||
| 3095 | return foldICmpEqIntrinsicWithConstant(Cmp, II, C); | ||||||||
| 3096 | |||||||||
| 3097 | Type *Ty = II->getType(); | ||||||||
| 3098 | unsigned BitWidth = C.getBitWidth(); | ||||||||
| 3099 | switch (II->getIntrinsicID()) { | ||||||||
| 3100 | case Intrinsic::ctlz: { | ||||||||
| 3101 | // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000 | ||||||||
| 3102 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { | ||||||||
| 3103 | unsigned Num = C.getLimitedValue(); | ||||||||
| 3104 | APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1); | ||||||||
| 3105 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, | ||||||||
| 3106 | II->getArgOperand(0), ConstantInt::get(Ty, Limit)); | ||||||||
| 3107 | } | ||||||||
| 3108 | |||||||||
| 3109 | // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111 | ||||||||
| 3110 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT && | ||||||||
| 3111 | C.uge(1) && C.ule(BitWidth)) { | ||||||||
| 3112 | unsigned Num = C.getLimitedValue(); | ||||||||
| 3113 | APInt Limit = APInt::getLowBitsSet(BitWidth, BitWidth - Num); | ||||||||
| 3114 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, | ||||||||
| 3115 | II->getArgOperand(0), ConstantInt::get(Ty, Limit)); | ||||||||
| 3116 | } | ||||||||
| 3117 | break; | ||||||||
| 3118 | } | ||||||||
| 3119 | case Intrinsic::cttz: { | ||||||||
| 3120 | // Limit to one use to ensure we don't increase instruction count. | ||||||||
| 3121 | if (!II->hasOneUse()) | ||||||||
| 3122 | return nullptr; | ||||||||
| 3123 | |||||||||
| 3124 | // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0 | ||||||||
| 3125 | if (Cmp.getPredicate() == ICmpInst::ICMP_UGT && C.ult(BitWidth)) { | ||||||||
| 3126 | APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1); | ||||||||
| 3127 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, | ||||||||
| 3128 | Builder.CreateAnd(II->getArgOperand(0), Mask), | ||||||||
| 3129 | ConstantInt::getNullValue(Ty)); | ||||||||
| 3130 | } | ||||||||
| 3131 | |||||||||
| 3132 | // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0 | ||||||||
| 3133 | if (Cmp.getPredicate() == ICmpInst::ICMP_ULT && | ||||||||
| 3134 | C.uge(1) && C.ule(BitWidth)) { | ||||||||
| 3135 | APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue()); | ||||||||
| 3136 | return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, | ||||||||
| 3137 | Builder.CreateAnd(II->getArgOperand(0), Mask), | ||||||||
| 3138 | ConstantInt::getNullValue(Ty)); | ||||||||
| 3139 | } | ||||||||
| 3140 | break; | ||||||||
| 3141 | } | ||||||||
| 3142 | default: | ||||||||
| 3143 | break; | ||||||||
| 3144 | } | ||||||||
| 3145 | |||||||||
| 3146 | return nullptr; | ||||||||
| 3147 | } | ||||||||
| 3148 | |||||||||
| 3149 | /// Handle icmp with constant (but not simple integer constant) RHS. | ||||||||
| 3150 | Instruction *InstCombiner::foldICmpInstWithConstantNotInt(ICmpInst &I) { | ||||||||
| 3151 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 3152 | Constant *RHSC = dyn_cast<Constant>(Op1); | ||||||||
| 3153 | Instruction *LHSI = dyn_cast<Instruction>(Op0); | ||||||||
| 3154 | if (!RHSC || !LHSI) | ||||||||
| 3155 | return nullptr; | ||||||||
| 3156 | |||||||||
| 3157 | switch (LHSI->getOpcode()) { | ||||||||
| 3158 | case Instruction::GetElementPtr: | ||||||||
| 3159 | // icmp pred GEP (P, int 0, int 0, int 0), null -> icmp pred P, null | ||||||||
| 3160 | if (RHSC->isNullValue() && | ||||||||
| 3161 | cast<GetElementPtrInst>(LHSI)->hasAllZeroIndices()) | ||||||||
| 3162 | return new ICmpInst( | ||||||||
| 3163 | I.getPredicate(), LHSI->getOperand(0), | ||||||||
| 3164 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | ||||||||
| 3165 | break; | ||||||||
| 3166 | case Instruction::PHI: | ||||||||
| 3167 | // Only fold icmp into the PHI if the phi and icmp are in the same | ||||||||
| 3168 | // block. If in the same block, we're encouraging jump threading. If | ||||||||
| 3169 | // not, we are just pessimizing the code by making an i1 phi. | ||||||||
| 3170 | if (LHSI->getParent() == I.getParent()) | ||||||||
| 3171 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | ||||||||
| 3172 | return NV; | ||||||||
| 3173 | break; | ||||||||
| 3174 | case Instruction::Select: { | ||||||||
| 3175 | // If either operand of the select is a constant, we can fold the | ||||||||
| 3176 | // comparison into the select arms, which will cause one to be | ||||||||
| 3177 | // constant folded and the select turned into a bitwise or. | ||||||||
| 3178 | Value *Op1 = nullptr, *Op2 = nullptr; | ||||||||
| 3179 | ConstantInt *CI = nullptr; | ||||||||
| 3180 | if (Constant *C = dyn_cast<Constant>(LHSI->getOperand(1))) { | ||||||||
| 3181 | Op1 = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); | ||||||||
| 3182 | CI = dyn_cast<ConstantInt>(Op1); | ||||||||
| 3183 | } | ||||||||
| 3184 | if (Constant *C = dyn_cast<Constant>(LHSI->getOperand(2))) { | ||||||||
| 3185 | Op2 = ConstantExpr::getICmp(I.getPredicate(), C, RHSC); | ||||||||
| 3186 | CI = dyn_cast<ConstantInt>(Op2); | ||||||||
| 3187 | } | ||||||||
| 3188 | |||||||||
| 3189 | // We only want to perform this transformation if it will not lead to | ||||||||
| 3190 | // additional code. This is true if either both sides of the select | ||||||||
| 3191 | // fold to a constant (in which case the icmp is replaced with a select | ||||||||
| 3192 | // which will usually simplify) or this is the only user of the | ||||||||
| 3193 | // select (in which case we are trading a select+icmp for a simpler | ||||||||
| 3194 | // select+icmp) or all uses of the select can be replaced based on | ||||||||
| 3195 | // dominance information ("Global cases"). | ||||||||
| 3196 | bool Transform = false; | ||||||||
| 3197 | if (Op1 && Op2) | ||||||||
| 3198 | Transform = true; | ||||||||
| 3199 | else if (Op1 || Op2) { | ||||||||
| 3200 | // Local case | ||||||||
| 3201 | if (LHSI->hasOneUse()) | ||||||||
| 3202 | Transform = true; | ||||||||
| 3203 | // Global cases | ||||||||
| 3204 | else if (CI && !CI->isZero()) | ||||||||
| 3205 | // When Op1 is constant try replacing select with second operand. | ||||||||
| 3206 | // Otherwise Op2 is constant and try replacing select with first | ||||||||
| 3207 | // operand. | ||||||||
| 3208 | Transform = | ||||||||
| 3209 | replacedSelectWithOperand(cast<SelectInst>(LHSI), &I, Op1 ? 2 : 1); | ||||||||
| 3210 | } | ||||||||
| 3211 | if (Transform) { | ||||||||
| 3212 | if (!Op1) | ||||||||
| 3213 | Op1 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(1), RHSC, | ||||||||
| 3214 | I.getName()); | ||||||||
| 3215 | if (!Op2) | ||||||||
| 3216 | Op2 = Builder.CreateICmp(I.getPredicate(), LHSI->getOperand(2), RHSC, | ||||||||
| 3217 | I.getName()); | ||||||||
| 3218 | return SelectInst::Create(LHSI->getOperand(0), Op1, Op2); | ||||||||
| 3219 | } | ||||||||
| 3220 | break; | ||||||||
| 3221 | } | ||||||||
| 3222 | case Instruction::IntToPtr: | ||||||||
| 3223 | // icmp pred inttoptr(X), null -> icmp pred X, 0 | ||||||||
| 3224 | if (RHSC->isNullValue() && | ||||||||
| 3225 | DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(0)->getType()) | ||||||||
| 3226 | return new ICmpInst( | ||||||||
| 3227 | I.getPredicate(), LHSI->getOperand(0), | ||||||||
| 3228 | Constant::getNullValue(LHSI->getOperand(0)->getType())); | ||||||||
| 3229 | break; | ||||||||
| 3230 | |||||||||
| 3231 | case Instruction::Load: | ||||||||
| 3232 | // Try to optimize things like "A[i] > 4" to index computations. | ||||||||
| 3233 | if (GetElementPtrInst *GEP = | ||||||||
| 3234 | dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) { | ||||||||
| 3235 | if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||||||
| 3236 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||||||
| 3237 | !cast<LoadInst>(LHSI)->isVolatile()) | ||||||||
| 3238 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | ||||||||
| 3239 | return Res; | ||||||||
| 3240 | } | ||||||||
| 3241 | break; | ||||||||
| 3242 | } | ||||||||
| 3243 | |||||||||
| 3244 | return nullptr; | ||||||||
| 3245 | } | ||||||||
| 3246 | |||||||||
| 3247 | /// Some comparisons can be simplified. | ||||||||
| 3248 | /// In this case, we are looking for comparisons that look like | ||||||||
| 3249 | /// a check for a lossy truncation. | ||||||||
| 3250 | /// Folds: | ||||||||
| 3251 | /// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask | ||||||||
| 3252 | /// Where Mask is some pattern that produces all-ones in low bits: | ||||||||
| 3253 | /// (-1 >> y) | ||||||||
| 3254 | /// ((-1 << y) >> y) <- non-canonical, has extra uses | ||||||||
| 3255 | /// ~(-1 << y) | ||||||||
| 3256 | /// ((1 << y) + (-1)) <- non-canonical, has extra uses | ||||||||
| 3257 | /// The Mask can be a constant, too. | ||||||||
| 3258 | /// For some predicates, the operands are commutative. | ||||||||
| 3259 | /// For others, x can only be on a specific side. | ||||||||
| 3260 | static Value *foldICmpWithLowBitMaskedVal(ICmpInst &I, | ||||||||
| 3261 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3262 | ICmpInst::Predicate SrcPred; | ||||||||
| 3263 | Value *X, *M, *Y; | ||||||||
| 3264 | auto m_VariableMask = m_CombineOr( | ||||||||
| 3265 | m_CombineOr(m_Not(m_Shl(m_AllOnes(), m_Value())), | ||||||||
| 3266 | m_Add(m_Shl(m_One(), m_Value()), m_AllOnes())), | ||||||||
| 3267 | m_CombineOr(m_LShr(m_AllOnes(), m_Value()), | ||||||||
| 3268 | m_LShr(m_Shl(m_AllOnes(), m_Value(Y)), m_Deferred(Y)))); | ||||||||
| 3269 | auto m_Mask = m_CombineOr(m_VariableMask, m_LowBitMask()); | ||||||||
| 3270 | if (!match(&I, m_c_ICmp(SrcPred, | ||||||||
| 3271 | m_c_And(m_CombineAnd(m_Mask, m_Value(M)), m_Value(X)), | ||||||||
| 3272 | m_Deferred(X)))) | ||||||||
| 3273 | return nullptr; | ||||||||
| 3274 | |||||||||
| 3275 | ICmpInst::Predicate DstPred; | ||||||||
| 3276 | switch (SrcPred) { | ||||||||
| 3277 | case ICmpInst::Predicate::ICMP_EQ: | ||||||||
| 3278 | // x & (-1 >> y) == x -> x u<= (-1 >> y) | ||||||||
| 3279 | DstPred = ICmpInst::Predicate::ICMP_ULE; | ||||||||
| 3280 | break; | ||||||||
| 3281 | case ICmpInst::Predicate::ICMP_NE: | ||||||||
| 3282 | // x & (-1 >> y) != x -> x u> (-1 >> y) | ||||||||
| 3283 | DstPred = ICmpInst::Predicate::ICMP_UGT; | ||||||||
| 3284 | break; | ||||||||
| 3285 | case ICmpInst::Predicate::ICMP_ULT: | ||||||||
| 3286 | // x & (-1 >> y) u< x -> x u> (-1 >> y) | ||||||||
| 3287 | // x u> x & (-1 >> y) -> x u> (-1 >> y) | ||||||||
| 3288 | DstPred = ICmpInst::Predicate::ICMP_UGT; | ||||||||
| 3289 | break; | ||||||||
| 3290 | case ICmpInst::Predicate::ICMP_UGE: | ||||||||
| 3291 | // x & (-1 >> y) u>= x -> x u<= (-1 >> y) | ||||||||
| 3292 | // x u<= x & (-1 >> y) -> x u<= (-1 >> y) | ||||||||
| 3293 | DstPred = ICmpInst::Predicate::ICMP_ULE; | ||||||||
| 3294 | break; | ||||||||
| 3295 | case ICmpInst::Predicate::ICMP_SLT: | ||||||||
| 3296 | // x & (-1 >> y) s< x -> x s> (-1 >> y) | ||||||||
| 3297 | // x s> x & (-1 >> y) -> x s> (-1 >> y) | ||||||||
| 3298 | if (!match(M, m_Constant())) // Can not do this fold with non-constant. | ||||||||
| 3299 | return nullptr; | ||||||||
| 3300 | if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. | ||||||||
| 3301 | return nullptr; | ||||||||
| 3302 | DstPred = ICmpInst::Predicate::ICMP_SGT; | ||||||||
| 3303 | break; | ||||||||
| 3304 | case ICmpInst::Predicate::ICMP_SGE: | ||||||||
| 3305 | // x & (-1 >> y) s>= x -> x s<= (-1 >> y) | ||||||||
| 3306 | // x s<= x & (-1 >> y) -> x s<= (-1 >> y) | ||||||||
| 3307 | if (!match(M, m_Constant())) // Can not do this fold with non-constant. | ||||||||
| 3308 | return nullptr; | ||||||||
| 3309 | if (!match(M, m_NonNegative())) // Must not have any -1 vector elements. | ||||||||
| 3310 | return nullptr; | ||||||||
| 3311 | DstPred = ICmpInst::Predicate::ICMP_SLE; | ||||||||
| 3312 | break; | ||||||||
| 3313 | case ICmpInst::Predicate::ICMP_SGT: | ||||||||
| 3314 | case ICmpInst::Predicate::ICMP_SLE: | ||||||||
| 3315 | return nullptr; | ||||||||
| 3316 | case ICmpInst::Predicate::ICMP_UGT: | ||||||||
| 3317 | case ICmpInst::Predicate::ICMP_ULE: | ||||||||
| 3318 | llvm_unreachable("Instsimplify took care of commut. variant")::llvm::llvm_unreachable_internal("Instsimplify took care of commut. variant" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3318); | ||||||||
| 3319 | break; | ||||||||
| 3320 | default: | ||||||||
| 3321 | llvm_unreachable("All possible folds are handled.")::llvm::llvm_unreachable_internal("All possible folds are handled." , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3321); | ||||||||
| 3322 | } | ||||||||
| 3323 | |||||||||
| 3324 | // The mask value may be a vector constant that has undefined elements. But it | ||||||||
| 3325 | // may not be safe to propagate those undefs into the new compare, so replace | ||||||||
| 3326 | // those elements by copying an existing, defined, and safe scalar constant. | ||||||||
| 3327 | Type *OpTy = M->getType(); | ||||||||
| 3328 | auto *VecC = dyn_cast<Constant>(M); | ||||||||
| 3329 | if (OpTy->isVectorTy() && VecC && VecC->containsUndefElement()) { | ||||||||
| 3330 | Constant *SafeReplacementConstant = nullptr; | ||||||||
| 3331 | for (unsigned i = 0, e = OpTy->getVectorNumElements(); i != e; ++i) { | ||||||||
| 3332 | if (!isa<UndefValue>(VecC->getAggregateElement(i))) { | ||||||||
| 3333 | SafeReplacementConstant = VecC->getAggregateElement(i); | ||||||||
| 3334 | break; | ||||||||
| 3335 | } | ||||||||
| 3336 | } | ||||||||
| 3337 | assert(SafeReplacementConstant && "Failed to find undef replacement")((SafeReplacementConstant && "Failed to find undef replacement" ) ? static_cast<void> (0) : __assert_fail ("SafeReplacementConstant && \"Failed to find undef replacement\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3337, __PRETTY_FUNCTION__)); | ||||||||
| 3338 | M = Constant::replaceUndefsWith(VecC, SafeReplacementConstant); | ||||||||
| 3339 | } | ||||||||
| 3340 | |||||||||
| 3341 | return Builder.CreateICmp(DstPred, X, M); | ||||||||
| 3342 | } | ||||||||
| 3343 | |||||||||
| 3344 | /// Some comparisons can be simplified. | ||||||||
| 3345 | /// In this case, we are looking for comparisons that look like | ||||||||
| 3346 | /// a check for a lossy signed truncation. | ||||||||
| 3347 | /// Folds: (MaskedBits is a constant.) | ||||||||
| 3348 | /// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x | ||||||||
| 3349 | /// Into: | ||||||||
| 3350 | /// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits) | ||||||||
| 3351 | /// Where KeptBits = bitwidth(%x) - MaskedBits | ||||||||
| 3352 | static Value * | ||||||||
| 3353 | foldICmpWithTruncSignExtendedVal(ICmpInst &I, | ||||||||
| 3354 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3355 | ICmpInst::Predicate SrcPred; | ||||||||
| 3356 | Value *X; | ||||||||
| 3357 | const APInt *C0, *C1; // FIXME: non-splats, potentially with undef. | ||||||||
| 3358 | // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use. | ||||||||
| 3359 | if (!match(&I, m_c_ICmp(SrcPred, | ||||||||
| 3360 | m_OneUse(m_AShr(m_Shl(m_Value(X), m_APInt(C0)), | ||||||||
| 3361 | m_APInt(C1))), | ||||||||
| 3362 | m_Deferred(X)))) | ||||||||
| 3363 | return nullptr; | ||||||||
| 3364 | |||||||||
| 3365 | // Potential handling of non-splats: for each element: | ||||||||
| 3366 | // * if both are undef, replace with constant 0. | ||||||||
| 3367 | // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0. | ||||||||
| 3368 | // * if both are not undef, and are different, bailout. | ||||||||
| 3369 | // * else, only one is undef, then pick the non-undef one. | ||||||||
| 3370 | |||||||||
| 3371 | // The shift amount must be equal. | ||||||||
| 3372 | if (*C0 != *C1) | ||||||||
| 3373 | return nullptr; | ||||||||
| 3374 | const APInt &MaskedBits = *C0; | ||||||||
| 3375 | assert(MaskedBits != 0 && "shift by zero should be folded away already.")((MaskedBits != 0 && "shift by zero should be folded away already." ) ? static_cast<void> (0) : __assert_fail ("MaskedBits != 0 && \"shift by zero should be folded away already.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3375, __PRETTY_FUNCTION__)); | ||||||||
| 3376 | |||||||||
| 3377 | ICmpInst::Predicate DstPred; | ||||||||
| 3378 | switch (SrcPred) { | ||||||||
| 3379 | case ICmpInst::Predicate::ICMP_EQ: | ||||||||
| 3380 | // ((%x << MaskedBits) a>> MaskedBits) == %x | ||||||||
| 3381 | // => | ||||||||
| 3382 | // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits) | ||||||||
| 3383 | DstPred = ICmpInst::Predicate::ICMP_ULT; | ||||||||
| 3384 | break; | ||||||||
| 3385 | case ICmpInst::Predicate::ICMP_NE: | ||||||||
| 3386 | // ((%x << MaskedBits) a>> MaskedBits) != %x | ||||||||
| 3387 | // => | ||||||||
| 3388 | // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits) | ||||||||
| 3389 | DstPred = ICmpInst::Predicate::ICMP_UGE; | ||||||||
| 3390 | break; | ||||||||
| 3391 | // FIXME: are more folds possible? | ||||||||
| 3392 | default: | ||||||||
| 3393 | return nullptr; | ||||||||
| 3394 | } | ||||||||
| 3395 | |||||||||
| 3396 | auto *XType = X->getType(); | ||||||||
| 3397 | const unsigned XBitWidth = XType->getScalarSizeInBits(); | ||||||||
| 3398 | const APInt BitWidth = APInt(XBitWidth, XBitWidth); | ||||||||
| 3399 | assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched")((BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched" ) ? static_cast<void> (0) : __assert_fail ("BitWidth.ugt(MaskedBits) && \"shifts should leave some bits untouched\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3399, __PRETTY_FUNCTION__)); | ||||||||
| 3400 | |||||||||
| 3401 | // KeptBits = bitwidth(%x) - MaskedBits | ||||||||
| 3402 | const APInt KeptBits = BitWidth - MaskedBits; | ||||||||
| 3403 | assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable")((KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable") ? static_cast<void> (0) : __assert_fail ("KeptBits.ugt(0) && KeptBits.ult(BitWidth) && \"unreachable\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3403, __PRETTY_FUNCTION__)); | ||||||||
| 3404 | // ICmpCst = (1 << KeptBits) | ||||||||
| 3405 | const APInt ICmpCst = APInt(XBitWidth, 1).shl(KeptBits); | ||||||||
| 3406 | assert(ICmpCst.isPowerOf2())((ICmpCst.isPowerOf2()) ? static_cast<void> (0) : __assert_fail ("ICmpCst.isPowerOf2()", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3406, __PRETTY_FUNCTION__)); | ||||||||
| 3407 | // AddCst = (1 << (KeptBits-1)) | ||||||||
| 3408 | const APInt AddCst = ICmpCst.lshr(1); | ||||||||
| 3409 | assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2())((AddCst.ult(ICmpCst) && AddCst.isPowerOf2()) ? static_cast <void> (0) : __assert_fail ("AddCst.ult(ICmpCst) && AddCst.isPowerOf2()" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3409, __PRETTY_FUNCTION__)); | ||||||||
| 3410 | |||||||||
| 3411 | // T0 = add %x, AddCst | ||||||||
| 3412 | Value *T0 = Builder.CreateAdd(X, ConstantInt::get(XType, AddCst)); | ||||||||
| 3413 | // T1 = T0 DstPred ICmpCst | ||||||||
| 3414 | Value *T1 = Builder.CreateICmp(DstPred, T0, ConstantInt::get(XType, ICmpCst)); | ||||||||
| 3415 | |||||||||
| 3416 | return T1; | ||||||||
| 3417 | } | ||||||||
| 3418 | |||||||||
| 3419 | // Given pattern: | ||||||||
| 3420 | // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0 | ||||||||
| 3421 | // we should move shifts to the same hand of 'and', i.e. rewrite as | ||||||||
| 3422 | // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x) | ||||||||
| 3423 | // We are only interested in opposite logical shifts here. | ||||||||
| 3424 | // One of the shifts can be truncated. | ||||||||
| 3425 | // If we can, we want to end up creating 'lshr' shift. | ||||||||
| 3426 | static Value * | ||||||||
| 3427 | foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ, | ||||||||
| 3428 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3429 | if (!I.isEquality() || !match(I.getOperand(1), m_Zero()) || | ||||||||
| 3430 | !I.getOperand(0)->hasOneUse()) | ||||||||
| 3431 | return nullptr; | ||||||||
| 3432 | |||||||||
| 3433 | auto m_AnyLogicalShift = m_LogicalShift(m_Value(), m_Value()); | ||||||||
| 3434 | |||||||||
| 3435 | // Look for an 'and' of two logical shifts, one of which may be truncated. | ||||||||
| 3436 | // We use m_TruncOrSelf() on the RHS to correctly handle commutative case. | ||||||||
| 3437 | Instruction *XShift, *MaybeTruncation, *YShift; | ||||||||
| 3438 | if (!match( | ||||||||
| |||||||||
| |||||||||
| 3439 | I.getOperand(0), | ||||||||
| 3440 | m_c_And(m_CombineAnd(m_AnyLogicalShift, m_Instruction(XShift)), | ||||||||
| 3441 | m_CombineAnd(m_TruncOrSelf(m_CombineAnd( | ||||||||
| 3442 | m_AnyLogicalShift, m_Instruction(YShift))), | ||||||||
| 3443 | m_Instruction(MaybeTruncation))))) | ||||||||
| 3444 | return nullptr; | ||||||||
| 3445 | |||||||||
| 3446 | // We potentially looked past 'trunc', but only when matching YShift, | ||||||||
| 3447 | // therefore YShift must have the widest type. | ||||||||
| 3448 | Instruction *WidestShift = YShift; | ||||||||
| 3449 | // Therefore XShift must have the shallowest type. | ||||||||
| 3450 | // Or they both have identical types if there was no truncation. | ||||||||
| 3451 | Instruction *NarrowestShift = XShift; | ||||||||
| 3452 | |||||||||
| 3453 | Type *WidestTy = WidestShift->getType(); | ||||||||
| 3454 | assert(NarrowestShift->getType() == I.getOperand(0)->getType() &&((NarrowestShift->getType() == I.getOperand(0)->getType () && "We did not look past any shifts while matching XShift though." ) ? static_cast<void> (0) : __assert_fail ("NarrowestShift->getType() == I.getOperand(0)->getType() && \"We did not look past any shifts while matching XShift though.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3455, __PRETTY_FUNCTION__)) | ||||||||
| 3455 | "We did not look past any shifts while matching XShift though.")((NarrowestShift->getType() == I.getOperand(0)->getType () && "We did not look past any shifts while matching XShift though." ) ? static_cast<void> (0) : __assert_fail ("NarrowestShift->getType() == I.getOperand(0)->getType() && \"We did not look past any shifts while matching XShift though.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3455, __PRETTY_FUNCTION__)); | ||||||||
| 3456 | bool HadTrunc = WidestTy != I.getOperand(0)->getType(); | ||||||||
| 3457 | |||||||||
| 3458 | // If YShift is a 'lshr', swap the shifts around. | ||||||||
| 3459 | if (match(YShift, m_LShr(m_Value(), m_Value()))) | ||||||||
| 3460 | std::swap(XShift, YShift); | ||||||||
| 3461 | |||||||||
| 3462 | // The shifts must be in opposite directions. | ||||||||
| 3463 | auto XShiftOpcode = XShift->getOpcode(); | ||||||||
| 3464 | if (XShiftOpcode == YShift->getOpcode()) | ||||||||
| 3465 | return nullptr; // Do not care about same-direction shifts here. | ||||||||
| 3466 | |||||||||
| 3467 | Value *X, *XShAmt, *Y, *YShAmt; | ||||||||
| 3468 | match(XShift, m_BinOp(m_Value(X), m_ZExtOrSelf(m_Value(XShAmt)))); | ||||||||
| 3469 | match(YShift, m_BinOp(m_Value(Y), m_ZExtOrSelf(m_Value(YShAmt)))); | ||||||||
| 3470 | |||||||||
| 3471 | // If one of the values being shifted is a constant, then we will end with | ||||||||
| 3472 | // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not, | ||||||||
| 3473 | // however, we will need to ensure that we won't increase instruction count. | ||||||||
| 3474 | if (!isa<Constant>(X) && !isa<Constant>(Y)) { | ||||||||
| 3475 | // At least one of the hands of the 'and' should be one-use shift. | ||||||||
| 3476 | if (!match(I.getOperand(0), | ||||||||
| 3477 | m_c_And(m_OneUse(m_AnyLogicalShift), m_Value()))) | ||||||||
| 3478 | return nullptr; | ||||||||
| 3479 | if (HadTrunc) { | ||||||||
| 3480 | // Due to the 'trunc', we will need to widen X. For that either the old | ||||||||
| 3481 | // 'trunc' or the shift amt in the non-truncated shift should be one-use. | ||||||||
| 3482 | if (!MaybeTruncation->hasOneUse() && | ||||||||
| 3483 | !NarrowestShift->getOperand(1)->hasOneUse()) | ||||||||
| 3484 | return nullptr; | ||||||||
| 3485 | } | ||||||||
| 3486 | } | ||||||||
| 3487 | |||||||||
| 3488 | // We have two shift amounts from two different shifts. The types of those | ||||||||
| 3489 | // shift amounts may not match. If that's the case let's bailout now. | ||||||||
| 3490 | if (XShAmt->getType() != YShAmt->getType()) | ||||||||
| |||||||||
| 3491 | return nullptr; | ||||||||
| 3492 | |||||||||
| 3493 | // Can we fold (XShAmt+YShAmt) ? | ||||||||
| 3494 | auto *NewShAmt = dyn_cast_or_null<Constant>( | ||||||||
| 3495 | SimplifyAddInst(XShAmt, YShAmt, /*isNSW=*/false, | ||||||||
| 3496 | /*isNUW=*/false, SQ.getWithInstruction(&I))); | ||||||||
| 3497 | if (!NewShAmt) | ||||||||
| 3498 | return nullptr; | ||||||||
| 3499 | NewShAmt = ConstantExpr::getZExtOrBitCast(NewShAmt, WidestTy); | ||||||||
| 3500 | unsigned WidestBitWidth = WidestTy->getScalarSizeInBits(); | ||||||||
| 3501 | |||||||||
| 3502 | // Is the new shift amount smaller than the bit width? | ||||||||
| 3503 | // FIXME: could also rely on ConstantRange. | ||||||||
| 3504 | if (!match(NewShAmt, | ||||||||
| 3505 | m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_ULT, | ||||||||
| 3506 | APInt(WidestBitWidth, WidestBitWidth)))) | ||||||||
| 3507 | return nullptr; | ||||||||
| 3508 | |||||||||
| 3509 | // An extra legality check is needed if we had trunc-of-lshr. | ||||||||
| 3510 | if (HadTrunc && match(WidestShift, m_LShr(m_Value(), m_Value()))) { | ||||||||
| 3511 | auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ, | ||||||||
| 3512 | WidestShift]() { | ||||||||
| 3513 | // It isn't obvious whether it's worth it to analyze non-constants here. | ||||||||
| 3514 | // Also, let's basically give up on non-splat cases, pessimizing vectors. | ||||||||
| 3515 | // If *any* of these preconditions matches we can perform the fold. | ||||||||
| 3516 | Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy() | ||||||||
| 3517 | ? NewShAmt->getSplatValue() | ||||||||
| 3518 | : NewShAmt; | ||||||||
| 3519 | // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold. | ||||||||
| 3520 | if (NewShAmtSplat && | ||||||||
| 3521 | (NewShAmtSplat->isNullValue() || | ||||||||
| 3522 | NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1)) | ||||||||
| 3523 | return true; | ||||||||
| 3524 | // We consider *min* leading zeros so a single outlier | ||||||||
| 3525 | // blocks the transform as opposed to allowing it. | ||||||||
| 3526 | if (auto *C = dyn_cast<Constant>(NarrowestShift->getOperand(0))) { | ||||||||
| 3527 | KnownBits Known = computeKnownBits(C, SQ.DL); | ||||||||
| 3528 | unsigned MinLeadZero = Known.countMinLeadingZeros(); | ||||||||
| 3529 | // If the value being shifted has at most lowest bit set we can fold. | ||||||||
| 3530 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; | ||||||||
| 3531 | if (MaxActiveBits <= 1) | ||||||||
| 3532 | return true; | ||||||||
| 3533 | // Precondition: NewShAmt u<= countLeadingZeros(C) | ||||||||
| 3534 | if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(MinLeadZero)) | ||||||||
| 3535 | return true; | ||||||||
| 3536 | } | ||||||||
| 3537 | if (auto *C = dyn_cast<Constant>(WidestShift->getOperand(0))) { | ||||||||
| 3538 | KnownBits Known = computeKnownBits(C, SQ.DL); | ||||||||
| 3539 | unsigned MinLeadZero = Known.countMinLeadingZeros(); | ||||||||
| 3540 | // If the value being shifted has at most lowest bit set we can fold. | ||||||||
| 3541 | unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero; | ||||||||
| 3542 | if (MaxActiveBits <= 1) | ||||||||
| 3543 | return true; | ||||||||
| 3544 | // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C) | ||||||||
| 3545 | if (NewShAmtSplat) { | ||||||||
| 3546 | APInt AdjNewShAmt = | ||||||||
| 3547 | (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger(); | ||||||||
| 3548 | if (AdjNewShAmt.ule(MinLeadZero)) | ||||||||
| 3549 | return true; | ||||||||
| 3550 | } | ||||||||
| 3551 | } | ||||||||
| 3552 | return false; // Can't tell if it's ok. | ||||||||
| 3553 | }; | ||||||||
| 3554 | if (!CanFold()) | ||||||||
| 3555 | return nullptr; | ||||||||
| 3556 | } | ||||||||
| 3557 | |||||||||
| 3558 | // All good, we can do this fold. | ||||||||
| 3559 | X = Builder.CreateZExt(X, WidestTy); | ||||||||
| 3560 | Y = Builder.CreateZExt(Y, WidestTy); | ||||||||
| 3561 | // The shift is the same that was for X. | ||||||||
| 3562 | Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr | ||||||||
| 3563 | ? Builder.CreateLShr(X, NewShAmt) | ||||||||
| 3564 | : Builder.CreateShl(X, NewShAmt); | ||||||||
| 3565 | Value *T1 = Builder.CreateAnd(T0, Y); | ||||||||
| 3566 | return Builder.CreateICmp(I.getPredicate(), T1, | ||||||||
| 3567 | Constant::getNullValue(WidestTy)); | ||||||||
| 3568 | } | ||||||||
| 3569 | |||||||||
| 3570 | /// Fold | ||||||||
| 3571 | /// (-1 u/ x) u< y | ||||||||
| 3572 | /// ((x * y) u/ x) != y | ||||||||
| 3573 | /// to | ||||||||
| 3574 | /// @llvm.umul.with.overflow(x, y) plus extraction of overflow bit | ||||||||
| 3575 | /// Note that the comparison is commutative, while inverted (u>=, ==) predicate | ||||||||
| 3576 | /// will mean that we are looking for the opposite answer. | ||||||||
| 3577 | Value *InstCombiner::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) { | ||||||||
| 3578 | ICmpInst::Predicate Pred; | ||||||||
| 3579 | Value *X, *Y; | ||||||||
| 3580 | Instruction *Mul; | ||||||||
| 3581 | bool NeedNegation; | ||||||||
| 3582 | // Look for: (-1 u/ x) u</u>= y | ||||||||
| 3583 | if (!I.isEquality() && | ||||||||
| 3584 | match(&I, m_c_ICmp(Pred, m_OneUse(m_UDiv(m_AllOnes(), m_Value(X))), | ||||||||
| 3585 | m_Value(Y)))) { | ||||||||
| 3586 | Mul = nullptr; | ||||||||
| 3587 | |||||||||
| 3588 | // Are we checking that overflow does not happen, or does happen? | ||||||||
| 3589 | switch (Pred) { | ||||||||
| 3590 | case ICmpInst::Predicate::ICMP_ULT: | ||||||||
| 3591 | NeedNegation = false; | ||||||||
| 3592 | break; // OK | ||||||||
| 3593 | case ICmpInst::Predicate::ICMP_UGE: | ||||||||
| 3594 | NeedNegation = true; | ||||||||
| 3595 | break; // OK | ||||||||
| 3596 | default: | ||||||||
| 3597 | return nullptr; // Wrong predicate. | ||||||||
| 3598 | } | ||||||||
| 3599 | } else // Look for: ((x * y) u/ x) !=/== y | ||||||||
| 3600 | if (I.isEquality() && | ||||||||
| 3601 | match(&I, m_c_ICmp(Pred, m_Value(Y), | ||||||||
| 3602 | m_OneUse(m_UDiv(m_CombineAnd(m_c_Mul(m_Deferred(Y), | ||||||||
| 3603 | m_Value(X)), | ||||||||
| 3604 | m_Instruction(Mul)), | ||||||||
| 3605 | m_Deferred(X)))))) { | ||||||||
| 3606 | NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ; | ||||||||
| 3607 | } else | ||||||||
| 3608 | return nullptr; | ||||||||
| 3609 | |||||||||
| 3610 | BuilderTy::InsertPointGuard Guard(Builder); | ||||||||
| 3611 | // If the pattern included (x * y), we'll want to insert new instructions | ||||||||
| 3612 | // right before that original multiplication so that we can replace it. | ||||||||
| 3613 | bool MulHadOtherUses = Mul && !Mul->hasOneUse(); | ||||||||
| 3614 | if (MulHadOtherUses) | ||||||||
| 3615 | Builder.SetInsertPoint(Mul); | ||||||||
| 3616 | |||||||||
| 3617 | Function *F = Intrinsic::getDeclaration( | ||||||||
| 3618 | I.getModule(), Intrinsic::umul_with_overflow, X->getType()); | ||||||||
| 3619 | CallInst *Call = Builder.CreateCall(F, {X, Y}, "umul"); | ||||||||
| 3620 | |||||||||
| 3621 | // If the multiplication was used elsewhere, to ensure that we don't leave | ||||||||
| 3622 | // "duplicate" instructions, replace uses of that original multiplication | ||||||||
| 3623 | // with the multiplication result from the with.overflow intrinsic. | ||||||||
| 3624 | if (MulHadOtherUses) | ||||||||
| 3625 | replaceInstUsesWith(*Mul, Builder.CreateExtractValue(Call, 0, "umul.val")); | ||||||||
| 3626 | |||||||||
| 3627 | Value *Res = Builder.CreateExtractValue(Call, 1, "umul.ov"); | ||||||||
| 3628 | if (NeedNegation) // This technically increases instruction count. | ||||||||
| 3629 | Res = Builder.CreateNot(Res, "umul.not.ov"); | ||||||||
| 3630 | |||||||||
| 3631 | return Res; | ||||||||
| 3632 | } | ||||||||
| 3633 | |||||||||
| 3634 | /// Try to fold icmp (binop), X or icmp X, (binop). | ||||||||
| 3635 | /// TODO: A large part of this logic is duplicated in InstSimplify's | ||||||||
| 3636 | /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code | ||||||||
| 3637 | /// duplication. | ||||||||
| 3638 | Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I, const SimplifyQuery &SQ) { | ||||||||
| 3639 | const SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||||||
| 3640 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 3641 | |||||||||
| 3642 | // Special logic for binary operators. | ||||||||
| 3643 | BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Op0); | ||||||||
| |||||||||
| 3644 | BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Op1); | ||||||||
| 3645 | if (!BO0
| ||||||||
| 3646 | return nullptr; | ||||||||
| 3647 | |||||||||
| 3648 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 3649 | Value *X; | ||||||||
| 3650 | |||||||||
| 3651 | // Convert add-with-unsigned-overflow comparisons into a 'not' with compare. | ||||||||
| 3652 | // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X | ||||||||
| 3653 | if (match(Op0, m_OneUse(m_c_Add(m_Specific(Op1), m_Value(X)))) && | ||||||||
| 3654 | (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) | ||||||||
| 3655 | return new ICmpInst(Pred, Builder.CreateNot(Op1), X); | ||||||||
| 3656 | // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0 | ||||||||
| 3657 | if (match(Op1, m_OneUse(m_c_Add(m_Specific(Op0), m_Value(X)))) && | ||||||||
| 3658 | (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) | ||||||||
| 3659 | return new ICmpInst(Pred, X, Builder.CreateNot(Op0)); | ||||||||
| 3660 | |||||||||
| 3661 | bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; | ||||||||
| 3662 | if (BO0
| ||||||||
| 3663 | NoOp0WrapProblem = | ||||||||
| 3664 | ICmpInst::isEquality(Pred) || | ||||||||
| 3665 | (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) || | ||||||||
| 3666 | (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap()); | ||||||||
| 3667 | if (BO1
| ||||||||
| 3668 | NoOp1WrapProblem = | ||||||||
| 3669 | ICmpInst::isEquality(Pred) || | ||||||||
| 3670 | (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) || | ||||||||
| 3671 | (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap()); | ||||||||
| 3672 | |||||||||
| 3673 | // Analyze the case when either Op0 or Op1 is an add instruction. | ||||||||
| 3674 | // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). | ||||||||
| 3675 | Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; | ||||||||
| 3676 | if (BO0
| ||||||||
| 3677 | A = BO0->getOperand(0); | ||||||||
| 3678 | B = BO0->getOperand(1); | ||||||||
| 3679 | } | ||||||||
| 3680 | if (BO1
| ||||||||
| 3681 | C = BO1->getOperand(0); | ||||||||
| 3682 | D = BO1->getOperand(1); | ||||||||
| 3683 | } | ||||||||
| 3684 | |||||||||
| 3685 | // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow. | ||||||||
| 3686 | // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow. | ||||||||
| 3687 | if ((A
| ||||||||
| 3688 | return new ICmpInst(Pred, A == Op1 ? B : A, | ||||||||
| 3689 | Constant::getNullValue(Op1->getType())); | ||||||||
| 3690 | |||||||||
| 3691 | // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow. | ||||||||
| 3692 | // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow. | ||||||||
| 3693 | if ((C
| ||||||||
| 3694 | return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()), | ||||||||
| 3695 | C == Op0 ? D : C); | ||||||||
| 3696 | |||||||||
| 3697 | // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow. | ||||||||
| 3698 | if (A
| ||||||||
| 3699 | NoOp1WrapProblem) { | ||||||||
| 3700 | // Determine Y and Z in the form icmp (X+Y), (X+Z). | ||||||||
| 3701 | Value *Y, *Z; | ||||||||
| 3702 | if (A == C) { | ||||||||
| 3703 | // C + B == C + D -> B == D | ||||||||
| 3704 | Y = B; | ||||||||
| 3705 | Z = D; | ||||||||
| 3706 | } else if (A == D) { | ||||||||
| 3707 | // D + B == C + D -> B == C | ||||||||
| 3708 | Y = B; | ||||||||
| 3709 | Z = C; | ||||||||
| 3710 | } else if (B == C) { | ||||||||
| 3711 | // A + C == C + D -> A == D | ||||||||
| 3712 | Y = A; | ||||||||
| 3713 | Z = D; | ||||||||
| 3714 | } else { | ||||||||
| 3715 | assert(B == D)((B == D) ? static_cast<void> (0) : __assert_fail ("B == D" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 3715, __PRETTY_FUNCTION__)); | ||||||||
| 3716 | // A + D == C + D -> A == C | ||||||||
| 3717 | Y = A; | ||||||||
| 3718 | Z = C; | ||||||||
| 3719 | } | ||||||||
| 3720 | return new ICmpInst(Pred, Y, Z); | ||||||||
| 3721 | } | ||||||||
| 3722 | |||||||||
| 3723 | // icmp slt (A + -1), Op1 -> icmp sle A, Op1 | ||||||||
| 3724 | if (A
| ||||||||
| 3725 | match(B, m_AllOnes())) | ||||||||
| 3726 | return new ICmpInst(CmpInst::ICMP_SLE, A, Op1); | ||||||||
| 3727 | |||||||||
| 3728 | // icmp sge (A + -1), Op1 -> icmp sgt A, Op1 | ||||||||
| 3729 | if (A
| ||||||||
| 3730 | match(B, m_AllOnes())) | ||||||||
| 3731 | return new ICmpInst(CmpInst::ICMP_SGT, A, Op1); | ||||||||
| 3732 | |||||||||
| 3733 | // icmp sle (A + 1), Op1 -> icmp slt A, Op1 | ||||||||
| 3734 | if (A
| ||||||||
| 3735 | return new ICmpInst(CmpInst::ICMP_SLT, A, Op1); | ||||||||
| 3736 | |||||||||
| 3737 | // icmp sgt (A + 1), Op1 -> icmp sge A, Op1 | ||||||||
| 3738 | if (A
| ||||||||
| 3739 | return new ICmpInst(CmpInst::ICMP_SGE, A, Op1); | ||||||||
| 3740 | |||||||||
| 3741 | // icmp sgt Op0, (C + -1) -> icmp sge Op0, C | ||||||||
| 3742 | if (C
| ||||||||
| 3743 | match(D, m_AllOnes())) | ||||||||
| 3744 | return new ICmpInst(CmpInst::ICMP_SGE, Op0, C); | ||||||||
| 3745 | |||||||||
| 3746 | // icmp sle Op0, (C + -1) -> icmp slt Op0, C | ||||||||
| 3747 | if (C
| ||||||||
| 3748 | match(D, m_AllOnes())) | ||||||||
| 3749 | return new ICmpInst(CmpInst::ICMP_SLT, Op0, C); | ||||||||
| 3750 | |||||||||
| 3751 | // icmp sge Op0, (C + 1) -> icmp sgt Op0, C | ||||||||
| 3752 | if (C
| ||||||||
| 3753 | return new ICmpInst(CmpInst::ICMP_SGT, Op0, C); | ||||||||
| 3754 | |||||||||
| 3755 | // icmp slt Op0, (C + 1) -> icmp sle Op0, C | ||||||||
| 3756 | if (C
| ||||||||
| 3757 | return new ICmpInst(CmpInst::ICMP_SLE, Op0, C); | ||||||||
| 3758 | |||||||||
| 3759 | // TODO: The subtraction-related identities shown below also hold, but | ||||||||
| 3760 | // canonicalization from (X -nuw 1) to (X + -1) means that the combinations | ||||||||
| 3761 | // wouldn't happen even if they were implemented. | ||||||||
| 3762 | // | ||||||||
| 3763 | // icmp ult (A - 1), Op1 -> icmp ule A, Op1 | ||||||||
| 3764 | // icmp uge (A - 1), Op1 -> icmp ugt A, Op1 | ||||||||
| 3765 | // icmp ugt Op0, (C - 1) -> icmp uge Op0, C | ||||||||
| 3766 | // icmp ule Op0, (C - 1) -> icmp ult Op0, C | ||||||||
| 3767 | |||||||||
| 3768 | // icmp ule (A + 1), Op0 -> icmp ult A, Op1 | ||||||||
| 3769 | if (A
| ||||||||
| 3770 | return new ICmpInst(CmpInst::ICMP_ULT, A, Op1); | ||||||||
| 3771 | |||||||||
| 3772 | // icmp ugt (A + 1), Op0 -> icmp uge A, Op1 | ||||||||
| 3773 | if (A
| ||||||||
| 3774 | return new ICmpInst(CmpInst::ICMP_UGE, A, Op1); | ||||||||
| 3775 | |||||||||
| 3776 | // icmp uge Op0, (C + 1) -> icmp ugt Op0, C | ||||||||
| 3777 | if (C
| ||||||||
| 3778 | return new ICmpInst(CmpInst::ICMP_UGT, Op0, C); | ||||||||
| 3779 | |||||||||
| 3780 | // icmp ult Op0, (C + 1) -> icmp ule Op0, C | ||||||||
| 3781 | if (C
| ||||||||
| 3782 | return new ICmpInst(CmpInst::ICMP_ULE, Op0, C); | ||||||||
| 3783 | |||||||||
| 3784 | // if C1 has greater magnitude than C2: | ||||||||
| 3785 | // icmp (A + C1), (C + C2) -> icmp (A + C3), C | ||||||||
| 3786 | // s.t. C3 = C1 - C2 | ||||||||
| 3787 | // | ||||||||
| 3788 | // if C2 has greater magnitude than C1: | ||||||||
| 3789 | // icmp (A + C1), (C + C2) -> icmp A, (C + C3) | ||||||||
| 3790 | // s.t. C3 = C2 - C1 | ||||||||
| 3791 | if (A
| ||||||||
| 3792 | (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) | ||||||||
| 3793 | if (ConstantInt *C1 = dyn_cast<ConstantInt>(B)) | ||||||||
| 3794 | if (ConstantInt *C2 = dyn_cast<ConstantInt>(D)) { | ||||||||
| 3795 | const APInt &AP1 = C1->getValue(); | ||||||||
| 3796 | const APInt &AP2 = C2->getValue(); | ||||||||
| 3797 | if (AP1.isNegative() == AP2.isNegative()) { | ||||||||
| 3798 | APInt AP1Abs = C1->getValue().abs(); | ||||||||
| 3799 | APInt AP2Abs = C2->getValue().abs(); | ||||||||
| 3800 | if (AP1Abs.uge(AP2Abs)) { | ||||||||
| 3801 | ConstantInt *C3 = Builder.getInt(AP1 - AP2); | ||||||||
| 3802 | Value *NewAdd = Builder.CreateNSWAdd(A, C3); | ||||||||
| 3803 | return new ICmpInst(Pred, NewAdd, C); | ||||||||
| 3804 | } else { | ||||||||
| 3805 | ConstantInt *C3 = Builder.getInt(AP2 - AP1); | ||||||||
| 3806 | Value *NewAdd = Builder.CreateNSWAdd(C, C3); | ||||||||
| 3807 | return new ICmpInst(Pred, A, NewAdd); | ||||||||
| 3808 | } | ||||||||
| 3809 | } | ||||||||
| 3810 | } | ||||||||
| 3811 | |||||||||
| 3812 | // Analyze the case when either Op0 or Op1 is a sub instruction. | ||||||||
| 3813 | // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null). | ||||||||
| 3814 | A = nullptr; | ||||||||
| 3815 | B = nullptr; | ||||||||
| 3816 | C = nullptr; | ||||||||
| 3817 | D = nullptr; | ||||||||
| 3818 | if (BO0
| ||||||||
| 3819 | A = BO0->getOperand(0); | ||||||||
| 3820 | B = BO0->getOperand(1); | ||||||||
| 3821 | } | ||||||||
| 3822 | if (BO1
| ||||||||
| 3823 | C = BO1->getOperand(0); | ||||||||
| 3824 | D = BO1->getOperand(1); | ||||||||
| 3825 | } | ||||||||
| 3826 | |||||||||
| 3827 | // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow. | ||||||||
| 3828 | if (A
| ||||||||
| 3829 | return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B); | ||||||||
| 3830 | // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow. | ||||||||
| 3831 | if (C
| ||||||||
| 3832 | return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType())); | ||||||||
| 3833 | |||||||||
| 3834 | // Convert sub-with-unsigned-overflow comparisons into a comparison of args. | ||||||||
| 3835 | // (A - B) u>/u<= A --> B u>/u<= A | ||||||||
| 3836 | if (A
| ||||||||
| 3837 | return new ICmpInst(Pred, B, A); | ||||||||
| 3838 | // C u</u>= (C - D) --> C u</u>= D | ||||||||
| 3839 | if (C
| ||||||||
| 3840 | return new ICmpInst(Pred, C, D); | ||||||||
| 3841 | // (A - B) u>=/u< A --> B u>/u<= A iff B != 0 | ||||||||
| 3842 | if (A
| ||||||||
| 3843 | isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||||||
| 3844 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A); | ||||||||
| 3845 | // C u<=/u> (C - D) --> C u</u>= D iff B != 0 | ||||||||
| 3846 | if (C
| ||||||||
| 3847 | isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT)) | ||||||||
| 3848 | return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D); | ||||||||
| 3849 | |||||||||
| 3850 | // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow. | ||||||||
| 3851 | if (B
| ||||||||
| 3852 | return new ICmpInst(Pred, A, C); | ||||||||
| 3853 | |||||||||
| 3854 | // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow. | ||||||||
| 3855 | if (A
| ||||||||
| 3856 | return new ICmpInst(Pred, D, B); | ||||||||
| 3857 | |||||||||
| 3858 | // icmp (0-X) < cst --> x > -cst | ||||||||
| 3859 | if (NoOp0WrapProblem
| ||||||||
| 3860 | Value *X; | ||||||||
| 3861 | if (match(BO0, m_Neg(m_Value(X)))) | ||||||||
| 3862 | if (Constant *RHSC = dyn_cast<Constant>(Op1)) | ||||||||
| 3863 | if (RHSC->isNotMinSignedValue()) | ||||||||
| 3864 | return new ICmpInst(I.getSwappedPredicate(), X, | ||||||||
| 3865 | ConstantExpr::getNeg(RHSC)); | ||||||||
| 3866 | } | ||||||||
| 3867 | |||||||||
| 3868 | BinaryOperator *SRem = nullptr; | ||||||||
| 3869 | // icmp (srem X, Y), Y | ||||||||
| 3870 | if (BO0
| ||||||||
| 3871 | SRem = BO0; | ||||||||
| 3872 | // icmp Y, (srem X, Y) | ||||||||
| 3873 | else if (BO1
| ||||||||
| 3874 | Op0 == BO1->getOperand(1)) | ||||||||
| 3875 | SRem = BO1; | ||||||||
| 3876 | if (SRem
| ||||||||
| 3877 | // We don't check hasOneUse to avoid increasing register pressure because | ||||||||
| 3878 | // the value we use is the same value this instruction was already using. | ||||||||
| 3879 | switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(Pred) : Pred) { | ||||||||
| 3880 | default: | ||||||||
| 3881 | break; | ||||||||
| 3882 | case ICmpInst::ICMP_EQ: | ||||||||
| 3883 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 3884 | case ICmpInst::ICMP_NE: | ||||||||
| 3885 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 3886 | case ICmpInst::ICMP_SGT: | ||||||||
| 3887 | case ICmpInst::ICMP_SGE: | ||||||||
| 3888 | return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(1), | ||||||||
| 3889 | Constant::getAllOnesValue(SRem->getType())); | ||||||||
| 3890 | case ICmpInst::ICMP_SLT: | ||||||||
| 3891 | case ICmpInst::ICMP_SLE: | ||||||||
| 3892 | return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(1), | ||||||||
| 3893 | Constant::getNullValue(SRem->getType())); | ||||||||
| 3894 | } | ||||||||
| 3895 | } | ||||||||
| 3896 | |||||||||
| 3897 | if (BO0
| ||||||||
| 3898 | BO1->hasOneUse() && BO0->getOperand(1) == BO1->getOperand(1)) { | ||||||||
| 3899 | switch (BO0->getOpcode()) { | ||||||||
| 3900 | default: | ||||||||
| 3901 | break; | ||||||||
| 3902 | case Instruction::Add: | ||||||||
| 3903 | case Instruction::Sub: | ||||||||
| 3904 | case Instruction::Xor: { | ||||||||
| 3905 | if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b | ||||||||
| 3906 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3907 | |||||||||
| 3908 | const APInt *C; | ||||||||
| 3909 | if (match(BO0->getOperand(1), m_APInt(C))) { | ||||||||
| 3910 | // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b | ||||||||
| 3911 | if (C->isSignMask()) { | ||||||||
| 3912 | ICmpInst::Predicate NewPred = | ||||||||
| 3913 | I.isSigned() ? I.getUnsignedPredicate() : I.getSignedPredicate(); | ||||||||
| 3914 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3915 | } | ||||||||
| 3916 | |||||||||
| 3917 | // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b | ||||||||
| 3918 | if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) { | ||||||||
| 3919 | ICmpInst::Predicate NewPred = | ||||||||
| 3920 | I.isSigned() ? I.getUnsignedPredicate() : I.getSignedPredicate(); | ||||||||
| 3921 | NewPred = I.getSwappedPredicate(NewPred); | ||||||||
| 3922 | return new ICmpInst(NewPred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3923 | } | ||||||||
| 3924 | } | ||||||||
| 3925 | break; | ||||||||
| 3926 | } | ||||||||
| 3927 | case Instruction::Mul: { | ||||||||
| 3928 | if (!I.isEquality()) | ||||||||
| 3929 | break; | ||||||||
| 3930 | |||||||||
| 3931 | const APInt *C; | ||||||||
| 3932 | if (match(BO0->getOperand(1), m_APInt(C)) && !C->isNullValue() && | ||||||||
| 3933 | !C->isOneValue()) { | ||||||||
| 3934 | // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask) | ||||||||
| 3935 | // Mask = -1 >> count-trailing-zeros(C). | ||||||||
| 3936 | if (unsigned TZs = C->countTrailingZeros()) { | ||||||||
| 3937 | Constant *Mask = ConstantInt::get( | ||||||||
| 3938 | BO0->getType(), | ||||||||
| 3939 | APInt::getLowBitsSet(C->getBitWidth(), C->getBitWidth() - TZs)); | ||||||||
| 3940 | Value *And1 = Builder.CreateAnd(BO0->getOperand(0), Mask); | ||||||||
| 3941 | Value *And2 = Builder.CreateAnd(BO1->getOperand(0), Mask); | ||||||||
| 3942 | return new ICmpInst(Pred, And1, And2); | ||||||||
| 3943 | } | ||||||||
| 3944 | // If there are no trailing zeros in the multiplier, just eliminate | ||||||||
| 3945 | // the multiplies (no masking is needed): | ||||||||
| 3946 | // icmp eq/ne (X * C), (Y * C) --> icmp eq/ne X, Y | ||||||||
| 3947 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3948 | } | ||||||||
| 3949 | break; | ||||||||
| 3950 | } | ||||||||
| 3951 | case Instruction::UDiv: | ||||||||
| 3952 | case Instruction::LShr: | ||||||||
| 3953 | if (I.isSigned() || !BO0->isExact() || !BO1->isExact()) | ||||||||
| 3954 | break; | ||||||||
| 3955 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3956 | |||||||||
| 3957 | case Instruction::SDiv: | ||||||||
| 3958 | if (!I.isEquality() || !BO0->isExact() || !BO1->isExact()) | ||||||||
| 3959 | break; | ||||||||
| 3960 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3961 | |||||||||
| 3962 | case Instruction::AShr: | ||||||||
| 3963 | if (!BO0->isExact() || !BO1->isExact()) | ||||||||
| 3964 | break; | ||||||||
| 3965 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3966 | |||||||||
| 3967 | case Instruction::Shl: { | ||||||||
| 3968 | bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap(); | ||||||||
| 3969 | bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap(); | ||||||||
| 3970 | if (!NUW && !NSW) | ||||||||
| 3971 | break; | ||||||||
| 3972 | if (!NSW && I.isSigned()) | ||||||||
| 3973 | break; | ||||||||
| 3974 | return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); | ||||||||
| 3975 | } | ||||||||
| 3976 | } | ||||||||
| 3977 | } | ||||||||
| 3978 | |||||||||
| 3979 | if (BO0
| ||||||||
| 3980 | // Transform A & (L - 1) `ult` L --> L != 0 | ||||||||
| 3981 | auto LSubOne = m_Add(m_Specific(Op1), m_AllOnes()); | ||||||||
| 3982 | auto BitwiseAnd = m_c_And(m_Value(), LSubOne); | ||||||||
| 3983 | |||||||||
| 3984 | if (match(BO0, BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) { | ||||||||
| 3985 | auto *Zero = Constant::getNullValue(BO0->getType()); | ||||||||
| 3986 | return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero); | ||||||||
| 3987 | } | ||||||||
| 3988 | } | ||||||||
| 3989 | |||||||||
| 3990 | if (Value *V
| ||||||||
| 3991 | return replaceInstUsesWith(I, V); | ||||||||
| 3992 | |||||||||
| 3993 | if (Value *V
| ||||||||
| 3994 | return replaceInstUsesWith(I, V); | ||||||||
| 3995 | |||||||||
| 3996 | if (Value *V
| ||||||||
| 3997 | return replaceInstUsesWith(I, V); | ||||||||
| 3998 | |||||||||
| 3999 | if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder)) | ||||||||
| 4000 | return replaceInstUsesWith(I, V); | ||||||||
| 4001 | |||||||||
| 4002 | return nullptr; | ||||||||
| 4003 | } | ||||||||
| 4004 | |||||||||
| 4005 | /// Fold icmp Pred min|max(X, Y), X. | ||||||||
| 4006 | static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) { | ||||||||
| 4007 | ICmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 4008 | Value *Op0 = Cmp.getOperand(0); | ||||||||
| 4009 | Value *X = Cmp.getOperand(1); | ||||||||
| 4010 | |||||||||
| 4011 | // Canonicalize minimum or maximum operand to LHS of the icmp. | ||||||||
| 4012 | if (match(X, m_c_SMin(m_Specific(Op0), m_Value())) || | ||||||||
| 4013 | match(X, m_c_SMax(m_Specific(Op0), m_Value())) || | ||||||||
| 4014 | match(X, m_c_UMin(m_Specific(Op0), m_Value())) || | ||||||||
| 4015 | match(X, m_c_UMax(m_Specific(Op0), m_Value()))) { | ||||||||
| 4016 | std::swap(Op0, X); | ||||||||
| 4017 | Pred = Cmp.getSwappedPredicate(); | ||||||||
| 4018 | } | ||||||||
| 4019 | |||||||||
| 4020 | Value *Y; | ||||||||
| 4021 | if (match(Op0, m_c_SMin(m_Specific(X), m_Value(Y)))) { | ||||||||
| 4022 | // smin(X, Y) == X --> X s<= Y | ||||||||
| 4023 | // smin(X, Y) s>= X --> X s<= Y | ||||||||
| 4024 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SGE) | ||||||||
| 4025 | return new ICmpInst(ICmpInst::ICMP_SLE, X, Y); | ||||||||
| 4026 | |||||||||
| 4027 | // smin(X, Y) != X --> X s> Y | ||||||||
| 4028 | // smin(X, Y) s< X --> X s> Y | ||||||||
| 4029 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SLT) | ||||||||
| 4030 | return new ICmpInst(ICmpInst::ICMP_SGT, X, Y); | ||||||||
| 4031 | |||||||||
| 4032 | // These cases should be handled in InstSimplify: | ||||||||
| 4033 | // smin(X, Y) s<= X --> true | ||||||||
| 4034 | // smin(X, Y) s> X --> false | ||||||||
| 4035 | return nullptr; | ||||||||
| 4036 | } | ||||||||
| 4037 | |||||||||
| 4038 | if (match(Op0, m_c_SMax(m_Specific(X), m_Value(Y)))) { | ||||||||
| 4039 | // smax(X, Y) == X --> X s>= Y | ||||||||
| 4040 | // smax(X, Y) s<= X --> X s>= Y | ||||||||
| 4041 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLE) | ||||||||
| 4042 | return new ICmpInst(ICmpInst::ICMP_SGE, X, Y); | ||||||||
| 4043 | |||||||||
| 4044 | // smax(X, Y) != X --> X s< Y | ||||||||
| 4045 | // smax(X, Y) s> X --> X s< Y | ||||||||
| 4046 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_SGT) | ||||||||
| 4047 | return new ICmpInst(ICmpInst::ICMP_SLT, X, Y); | ||||||||
| 4048 | |||||||||
| 4049 | // These cases should be handled in InstSimplify: | ||||||||
| 4050 | // smax(X, Y) s>= X --> true | ||||||||
| 4051 | // smax(X, Y) s< X --> false | ||||||||
| 4052 | return nullptr; | ||||||||
| 4053 | } | ||||||||
| 4054 | |||||||||
| 4055 | if (match(Op0, m_c_UMin(m_Specific(X), m_Value(Y)))) { | ||||||||
| 4056 | // umin(X, Y) == X --> X u<= Y | ||||||||
| 4057 | // umin(X, Y) u>= X --> X u<= Y | ||||||||
| 4058 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_UGE) | ||||||||
| 4059 | return new ICmpInst(ICmpInst::ICMP_ULE, X, Y); | ||||||||
| 4060 | |||||||||
| 4061 | // umin(X, Y) != X --> X u> Y | ||||||||
| 4062 | // umin(X, Y) u< X --> X u> Y | ||||||||
| 4063 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT) | ||||||||
| 4064 | return new ICmpInst(ICmpInst::ICMP_UGT, X, Y); | ||||||||
| 4065 | |||||||||
| 4066 | // These cases should be handled in InstSimplify: | ||||||||
| 4067 | // umin(X, Y) u<= X --> true | ||||||||
| 4068 | // umin(X, Y) u> X --> false | ||||||||
| 4069 | return nullptr; | ||||||||
| 4070 | } | ||||||||
| 4071 | |||||||||
| 4072 | if (match(Op0, m_c_UMax(m_Specific(X), m_Value(Y)))) { | ||||||||
| 4073 | // umax(X, Y) == X --> X u>= Y | ||||||||
| 4074 | // umax(X, Y) u<= X --> X u>= Y | ||||||||
| 4075 | if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_ULE) | ||||||||
| 4076 | return new ICmpInst(ICmpInst::ICMP_UGE, X, Y); | ||||||||
| 4077 | |||||||||
| 4078 | // umax(X, Y) != X --> X u< Y | ||||||||
| 4079 | // umax(X, Y) u> X --> X u< Y | ||||||||
| 4080 | if (Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_UGT) | ||||||||
| 4081 | return new ICmpInst(ICmpInst::ICMP_ULT, X, Y); | ||||||||
| 4082 | |||||||||
| 4083 | // These cases should be handled in InstSimplify: | ||||||||
| 4084 | // umax(X, Y) u>= X --> true | ||||||||
| 4085 | // umax(X, Y) u< X --> false | ||||||||
| 4086 | return nullptr; | ||||||||
| 4087 | } | ||||||||
| 4088 | |||||||||
| 4089 | return nullptr; | ||||||||
| 4090 | } | ||||||||
| 4091 | |||||||||
| 4092 | Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) { | ||||||||
| 4093 | if (!I.isEquality()) | ||||||||
| 4094 | return nullptr; | ||||||||
| 4095 | |||||||||
| 4096 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 4097 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 4098 | Value *A, *B, *C, *D; | ||||||||
| 4099 | if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) { | ||||||||
| 4100 | if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0 | ||||||||
| 4101 | Value *OtherVal = A == Op1 ? B : A; | ||||||||
| 4102 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); | ||||||||
| 4103 | } | ||||||||
| 4104 | |||||||||
| 4105 | if (match(Op1, m_Xor(m_Value(C), m_Value(D)))) { | ||||||||
| 4106 | // A^c1 == C^c2 --> A == C^(c1^c2) | ||||||||
| 4107 | ConstantInt *C1, *C2; | ||||||||
| 4108 | if (match(B, m_ConstantInt(C1)) && match(D, m_ConstantInt(C2)) && | ||||||||
| 4109 | Op1->hasOneUse()) { | ||||||||
| 4110 | Constant *NC = Builder.getInt(C1->getValue() ^ C2->getValue()); | ||||||||
| 4111 | Value *Xor = Builder.CreateXor(C, NC); | ||||||||
| 4112 | return new ICmpInst(Pred, A, Xor); | ||||||||
| 4113 | } | ||||||||
| 4114 | |||||||||
| 4115 | // A^B == A^D -> B == D | ||||||||
| 4116 | if (A == C) | ||||||||
| 4117 | return new ICmpInst(Pred, B, D); | ||||||||
| 4118 | if (A == D) | ||||||||
| 4119 | return new ICmpInst(Pred, B, C); | ||||||||
| 4120 | if (B == C) | ||||||||
| 4121 | return new ICmpInst(Pred, A, D); | ||||||||
| 4122 | if (B == D) | ||||||||
| 4123 | return new ICmpInst(Pred, A, C); | ||||||||
| 4124 | } | ||||||||
| 4125 | } | ||||||||
| 4126 | |||||||||
| 4127 | if (match(Op1, m_Xor(m_Value(A), m_Value(B))) && (A == Op0 || B == Op0)) { | ||||||||
| 4128 | // A == (A^B) -> B == 0 | ||||||||
| 4129 | Value *OtherVal = A == Op0 ? B : A; | ||||||||
| 4130 | return new ICmpInst(Pred, OtherVal, Constant::getNullValue(A->getType())); | ||||||||
| 4131 | } | ||||||||
| 4132 | |||||||||
| 4133 | // (X&Z) == (Y&Z) -> (X^Y) & Z == 0 | ||||||||
| 4134 | if (match(Op0, m_OneUse(m_And(m_Value(A), m_Value(B)))) && | ||||||||
| 4135 | match(Op1, m_OneUse(m_And(m_Value(C), m_Value(D))))) { | ||||||||
| 4136 | Value *X = nullptr, *Y = nullptr, *Z = nullptr; | ||||||||
| 4137 | |||||||||
| 4138 | if (A == C) { | ||||||||
| 4139 | X = B; | ||||||||
| 4140 | Y = D; | ||||||||
| 4141 | Z = A; | ||||||||
| 4142 | } else if (A == D) { | ||||||||
| 4143 | X = B; | ||||||||
| 4144 | Y = C; | ||||||||
| 4145 | Z = A; | ||||||||
| 4146 | } else if (B == C) { | ||||||||
| 4147 | X = A; | ||||||||
| 4148 | Y = D; | ||||||||
| 4149 | Z = B; | ||||||||
| 4150 | } else if (B == D) { | ||||||||
| 4151 | X = A; | ||||||||
| 4152 | Y = C; | ||||||||
| 4153 | Z = B; | ||||||||
| 4154 | } | ||||||||
| 4155 | |||||||||
| 4156 | if (X) { // Build (X^Y) & Z | ||||||||
| 4157 | Op1 = Builder.CreateXor(X, Y); | ||||||||
| 4158 | Op1 = Builder.CreateAnd(Op1, Z); | ||||||||
| 4159 | return new ICmpInst(Pred, Op1, Constant::getNullValue(Op1->getType())); | ||||||||
| 4160 | } | ||||||||
| 4161 | } | ||||||||
| 4162 | |||||||||
| 4163 | // Transform (zext A) == (B & (1<<X)-1) --> A == (trunc B) | ||||||||
| 4164 | // and (B & (1<<X)-1) == (zext A) --> A == (trunc B) | ||||||||
| 4165 | ConstantInt *Cst1; | ||||||||
| 4166 | if ((Op0->hasOneUse() && match(Op0, m_ZExt(m_Value(A))) && | ||||||||
| 4167 | match(Op1, m_And(m_Value(B), m_ConstantInt(Cst1)))) || | ||||||||
| 4168 | (Op1->hasOneUse() && match(Op0, m_And(m_Value(B), m_ConstantInt(Cst1))) && | ||||||||
| 4169 | match(Op1, m_ZExt(m_Value(A))))) { | ||||||||
| 4170 | APInt Pow2 = Cst1->getValue() + 1; | ||||||||
| 4171 | if (Pow2.isPowerOf2() && isa<IntegerType>(A->getType()) && | ||||||||
| 4172 | Pow2.logBase2() == cast<IntegerType>(A->getType())->getBitWidth()) | ||||||||
| 4173 | return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType())); | ||||||||
| 4174 | } | ||||||||
| 4175 | |||||||||
| 4176 | // (A >> C) == (B >> C) --> (A^B) u< (1 << C) | ||||||||
| 4177 | // For lshr and ashr pairs. | ||||||||
| 4178 | if ((match(Op0, m_OneUse(m_LShr(m_Value(A), m_ConstantInt(Cst1)))) && | ||||||||
| 4179 | match(Op1, m_OneUse(m_LShr(m_Value(B), m_Specific(Cst1))))) || | ||||||||
| 4180 | (match(Op0, m_OneUse(m_AShr(m_Value(A), m_ConstantInt(Cst1)))) && | ||||||||
| 4181 | match(Op1, m_OneUse(m_AShr(m_Value(B), m_Specific(Cst1)))))) { | ||||||||
| 4182 | unsigned TypeBits = Cst1->getBitWidth(); | ||||||||
| 4183 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | ||||||||
| 4184 | if (ShAmt < TypeBits && ShAmt != 0) { | ||||||||
| 4185 | ICmpInst::Predicate NewPred = | ||||||||
| 4186 | Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT; | ||||||||
| 4187 | Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); | ||||||||
| 4188 | APInt CmpVal = APInt::getOneBitSet(TypeBits, ShAmt); | ||||||||
| 4189 | return new ICmpInst(NewPred, Xor, Builder.getInt(CmpVal)); | ||||||||
| 4190 | } | ||||||||
| 4191 | } | ||||||||
| 4192 | |||||||||
| 4193 | // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0 | ||||||||
| 4194 | if (match(Op0, m_OneUse(m_Shl(m_Value(A), m_ConstantInt(Cst1)))) && | ||||||||
| 4195 | match(Op1, m_OneUse(m_Shl(m_Value(B), m_Specific(Cst1))))) { | ||||||||
| 4196 | unsigned TypeBits = Cst1->getBitWidth(); | ||||||||
| 4197 | unsigned ShAmt = (unsigned)Cst1->getLimitedValue(TypeBits); | ||||||||
| 4198 | if (ShAmt < TypeBits && ShAmt != 0) { | ||||||||
| 4199 | Value *Xor = Builder.CreateXor(A, B, I.getName() + ".unshifted"); | ||||||||
| 4200 | APInt AndVal = APInt::getLowBitsSet(TypeBits, TypeBits - ShAmt); | ||||||||
| 4201 | Value *And = Builder.CreateAnd(Xor, Builder.getInt(AndVal), | ||||||||
| 4202 | I.getName() + ".mask"); | ||||||||
| 4203 | return new ICmpInst(Pred, And, Constant::getNullValue(Cst1->getType())); | ||||||||
| 4204 | } | ||||||||
| 4205 | } | ||||||||
| 4206 | |||||||||
| 4207 | // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to | ||||||||
| 4208 | // "icmp (and X, mask), cst" | ||||||||
| 4209 | uint64_t ShAmt = 0; | ||||||||
| 4210 | if (Op0->hasOneUse() && | ||||||||
| 4211 | match(Op0, m_Trunc(m_OneUse(m_LShr(m_Value(A), m_ConstantInt(ShAmt))))) && | ||||||||
| 4212 | match(Op1, m_ConstantInt(Cst1)) && | ||||||||
| 4213 | // Only do this when A has multiple uses. This is most important to do | ||||||||
| 4214 | // when it exposes other optimizations. | ||||||||
| 4215 | !A->hasOneUse()) { | ||||||||
| 4216 | unsigned ASize = cast<IntegerType>(A->getType())->getPrimitiveSizeInBits(); | ||||||||
| 4217 | |||||||||
| 4218 | if (ShAmt < ASize) { | ||||||||
| 4219 | APInt MaskV = | ||||||||
| 4220 | APInt::getLowBitsSet(ASize, Op0->getType()->getPrimitiveSizeInBits()); | ||||||||
| 4221 | MaskV <<= ShAmt; | ||||||||
| 4222 | |||||||||
| 4223 | APInt CmpV = Cst1->getValue().zext(ASize); | ||||||||
| 4224 | CmpV <<= ShAmt; | ||||||||
| 4225 | |||||||||
| 4226 | Value *Mask = Builder.CreateAnd(A, Builder.getInt(MaskV)); | ||||||||
| 4227 | return new ICmpInst(Pred, Mask, Builder.getInt(CmpV)); | ||||||||
| 4228 | } | ||||||||
| 4229 | } | ||||||||
| 4230 | |||||||||
| 4231 | // If both operands are byte-swapped or bit-reversed, just compare the | ||||||||
| 4232 | // original values. | ||||||||
| 4233 | // TODO: Move this to a function similar to foldICmpIntrinsicWithConstant() | ||||||||
| 4234 | // and handle more intrinsics. | ||||||||
| 4235 | if ((match(Op0, m_BSwap(m_Value(A))) && match(Op1, m_BSwap(m_Value(B)))) || | ||||||||
| 4236 | (match(Op0, m_BitReverse(m_Value(A))) && | ||||||||
| 4237 | match(Op1, m_BitReverse(m_Value(B))))) | ||||||||
| 4238 | return new ICmpInst(Pred, A, B); | ||||||||
| 4239 | |||||||||
| 4240 | // Canonicalize checking for a power-of-2-or-zero value: | ||||||||
| 4241 | // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants) | ||||||||
| 4242 | // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants) | ||||||||
| 4243 | if (!match(Op0, m_OneUse(m_c_And(m_Add(m_Value(A), m_AllOnes()), | ||||||||
| 4244 | m_Deferred(A)))) || | ||||||||
| 4245 | !match(Op1, m_ZeroInt())) | ||||||||
| 4246 | A = nullptr; | ||||||||
| 4247 | |||||||||
| 4248 | // (A & -A) == A --> ctpop(A) < 2 (four commuted variants) | ||||||||
| 4249 | // (-A & A) != A --> ctpop(A) > 1 (four commuted variants) | ||||||||
| 4250 | if (match(Op0, m_OneUse(m_c_And(m_Neg(m_Specific(Op1)), m_Specific(Op1))))) | ||||||||
| 4251 | A = Op1; | ||||||||
| 4252 | else if (match(Op1, | ||||||||
| 4253 | m_OneUse(m_c_And(m_Neg(m_Specific(Op0)), m_Specific(Op0))))) | ||||||||
| 4254 | A = Op0; | ||||||||
| 4255 | |||||||||
| 4256 | if (A) { | ||||||||
| 4257 | Type *Ty = A->getType(); | ||||||||
| 4258 | CallInst *CtPop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, A); | ||||||||
| 4259 | return Pred == ICmpInst::ICMP_EQ | ||||||||
| 4260 | ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop, ConstantInt::get(Ty, 2)) | ||||||||
| 4261 | : new ICmpInst(ICmpInst::ICMP_UGT, CtPop, ConstantInt::get(Ty, 1)); | ||||||||
| 4262 | } | ||||||||
| 4263 | |||||||||
| 4264 | return nullptr; | ||||||||
| 4265 | } | ||||||||
| 4266 | |||||||||
| 4267 | static Instruction *foldICmpWithZextOrSext(ICmpInst &ICmp, | ||||||||
| 4268 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 4269 | assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0")((isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0" ) ? static_cast<void> (0) : __assert_fail ("isa<CastInst>(ICmp.getOperand(0)) && \"Expected cast for operand 0\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4269, __PRETTY_FUNCTION__)); | ||||||||
| 4270 | auto *CastOp0 = cast<CastInst>(ICmp.getOperand(0)); | ||||||||
| 4271 | Value *X; | ||||||||
| 4272 | if (!match(CastOp0, m_ZExtOrSExt(m_Value(X)))) | ||||||||
| 4273 | return nullptr; | ||||||||
| 4274 | |||||||||
| 4275 | bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt; | ||||||||
| 4276 | bool IsSignedCmp = ICmp.isSigned(); | ||||||||
| 4277 | if (auto *CastOp1 = dyn_cast<CastInst>(ICmp.getOperand(1))) { | ||||||||
| 4278 | // If the signedness of the two casts doesn't agree (i.e. one is a sext | ||||||||
| 4279 | // and the other is a zext), then we can't handle this. | ||||||||
| 4280 | // TODO: This is too strict. We can handle some predicates (equality?). | ||||||||
| 4281 | if (CastOp0->getOpcode() != CastOp1->getOpcode()) | ||||||||
| 4282 | return nullptr; | ||||||||
| 4283 | |||||||||
| 4284 | // Not an extension from the same type? | ||||||||
| 4285 | Value *Y = CastOp1->getOperand(0); | ||||||||
| 4286 | Type *XTy = X->getType(), *YTy = Y->getType(); | ||||||||
| 4287 | if (XTy != YTy) { | ||||||||
| 4288 | // One of the casts must have one use because we are creating a new cast. | ||||||||
| 4289 | if (!CastOp0->hasOneUse() && !CastOp1->hasOneUse()) | ||||||||
| 4290 | return nullptr; | ||||||||
| 4291 | // Extend the narrower operand to the type of the wider operand. | ||||||||
| 4292 | if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits()) | ||||||||
| 4293 | X = Builder.CreateCast(CastOp0->getOpcode(), X, YTy); | ||||||||
| 4294 | else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits()) | ||||||||
| 4295 | Y = Builder.CreateCast(CastOp0->getOpcode(), Y, XTy); | ||||||||
| 4296 | else | ||||||||
| 4297 | return nullptr; | ||||||||
| 4298 | } | ||||||||
| 4299 | |||||||||
| 4300 | // (zext X) == (zext Y) --> X == Y | ||||||||
| 4301 | // (sext X) == (sext Y) --> X == Y | ||||||||
| 4302 | if (ICmp.isEquality()) | ||||||||
| 4303 | return new ICmpInst(ICmp.getPredicate(), X, Y); | ||||||||
| 4304 | |||||||||
| 4305 | // A signed comparison of sign extended values simplifies into a | ||||||||
| 4306 | // signed comparison. | ||||||||
| 4307 | if (IsSignedCmp && IsSignedExt) | ||||||||
| 4308 | return new ICmpInst(ICmp.getPredicate(), X, Y); | ||||||||
| 4309 | |||||||||
| 4310 | // The other three cases all fold into an unsigned comparison. | ||||||||
| 4311 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y); | ||||||||
| 4312 | } | ||||||||
| 4313 | |||||||||
| 4314 | // Below here, we are only folding a compare with constant. | ||||||||
| 4315 | auto *C = dyn_cast<Constant>(ICmp.getOperand(1)); | ||||||||
| 4316 | if (!C) | ||||||||
| 4317 | return nullptr; | ||||||||
| 4318 | |||||||||
| 4319 | // Compute the constant that would happen if we truncated to SrcTy then | ||||||||
| 4320 | // re-extended to DestTy. | ||||||||
| 4321 | Type *SrcTy = CastOp0->getSrcTy(); | ||||||||
| 4322 | Type *DestTy = CastOp0->getDestTy(); | ||||||||
| 4323 | Constant *Res1 = ConstantExpr::getTrunc(C, SrcTy); | ||||||||
| 4324 | Constant *Res2 = ConstantExpr::getCast(CastOp0->getOpcode(), Res1, DestTy); | ||||||||
| 4325 | |||||||||
| 4326 | // If the re-extended constant didn't change... | ||||||||
| 4327 | if (Res2 == C) { | ||||||||
| 4328 | if (ICmp.isEquality()) | ||||||||
| 4329 | return new ICmpInst(ICmp.getPredicate(), X, Res1); | ||||||||
| 4330 | |||||||||
| 4331 | // A signed comparison of sign extended values simplifies into a | ||||||||
| 4332 | // signed comparison. | ||||||||
| 4333 | if (IsSignedExt && IsSignedCmp) | ||||||||
| 4334 | return new ICmpInst(ICmp.getPredicate(), X, Res1); | ||||||||
| 4335 | |||||||||
| 4336 | // The other three cases all fold into an unsigned comparison. | ||||||||
| 4337 | return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res1); | ||||||||
| 4338 | } | ||||||||
| 4339 | |||||||||
| 4340 | // The re-extended constant changed, partly changed (in the case of a vector), | ||||||||
| 4341 | // or could not be determined to be equal (in the case of a constant | ||||||||
| 4342 | // expression), so the constant cannot be represented in the shorter type. | ||||||||
| 4343 | // All the cases that fold to true or false will have already been handled | ||||||||
| 4344 | // by SimplifyICmpInst, so only deal with the tricky case. | ||||||||
| 4345 | if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(C)) | ||||||||
| 4346 | return nullptr; | ||||||||
| 4347 | |||||||||
| 4348 | // Is source op positive? | ||||||||
| 4349 | // icmp ult (sext X), C --> icmp sgt X, -1 | ||||||||
| 4350 | if (ICmp.getPredicate() == ICmpInst::ICMP_ULT) | ||||||||
| 4351 | return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(SrcTy)); | ||||||||
| 4352 | |||||||||
| 4353 | // Is source op negative? | ||||||||
| 4354 | // icmp ugt (sext X), C --> icmp slt X, 0 | ||||||||
| 4355 | assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!")((ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!" ) ? static_cast<void> (0) : __assert_fail ("ICmp.getPredicate() == ICmpInst::ICMP_UGT && \"ICmp should be folded!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4355, __PRETTY_FUNCTION__)); | ||||||||
| 4356 | return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(SrcTy)); | ||||||||
| 4357 | } | ||||||||
| 4358 | |||||||||
| 4359 | /// Handle icmp (cast x), (cast or constant). | ||||||||
| 4360 | Instruction *InstCombiner::foldICmpWithCastOp(ICmpInst &ICmp) { | ||||||||
| 4361 | auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0)); | ||||||||
| 4362 | if (!CastOp0) | ||||||||
| 4363 | return nullptr; | ||||||||
| 4364 | if (!isa<Constant>(ICmp.getOperand(1)) && !isa<CastInst>(ICmp.getOperand(1))) | ||||||||
| 4365 | return nullptr; | ||||||||
| 4366 | |||||||||
| 4367 | Value *Op0Src = CastOp0->getOperand(0); | ||||||||
| 4368 | Type *SrcTy = CastOp0->getSrcTy(); | ||||||||
| 4369 | Type *DestTy = CastOp0->getDestTy(); | ||||||||
| 4370 | |||||||||
| 4371 | // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the | ||||||||
| 4372 | // integer type is the same size as the pointer type. | ||||||||
| 4373 | auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) { | ||||||||
| 4374 | if (isa<VectorType>(SrcTy)) { | ||||||||
| 4375 | SrcTy = cast<VectorType>(SrcTy)->getElementType(); | ||||||||
| 4376 | DestTy = cast<VectorType>(DestTy)->getElementType(); | ||||||||
| 4377 | } | ||||||||
| 4378 | return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth(); | ||||||||
| 4379 | }; | ||||||||
| 4380 | if (CastOp0->getOpcode() == Instruction::PtrToInt && | ||||||||
| 4381 | CompatibleSizes(SrcTy, DestTy)) { | ||||||||
| 4382 | Value *NewOp1 = nullptr; | ||||||||
| 4383 | if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(ICmp.getOperand(1))) { | ||||||||
| 4384 | Value *PtrSrc = PtrToIntOp1->getOperand(0); | ||||||||
| 4385 | if (PtrSrc->getType()->getPointerAddressSpace() == | ||||||||
| 4386 | Op0Src->getType()->getPointerAddressSpace()) { | ||||||||
| 4387 | NewOp1 = PtrToIntOp1->getOperand(0); | ||||||||
| 4388 | // If the pointer types don't match, insert a bitcast. | ||||||||
| 4389 | if (Op0Src->getType() != NewOp1->getType()) | ||||||||
| 4390 | NewOp1 = Builder.CreateBitCast(NewOp1, Op0Src->getType()); | ||||||||
| 4391 | } | ||||||||
| 4392 | } else if (auto *RHSC = dyn_cast<Constant>(ICmp.getOperand(1))) { | ||||||||
| 4393 | NewOp1 = ConstantExpr::getIntToPtr(RHSC, SrcTy); | ||||||||
| 4394 | } | ||||||||
| 4395 | |||||||||
| 4396 | if (NewOp1) | ||||||||
| 4397 | return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1); | ||||||||
| 4398 | } | ||||||||
| 4399 | |||||||||
| 4400 | return foldICmpWithZextOrSext(ICmp, Builder); | ||||||||
| 4401 | } | ||||||||
| 4402 | |||||||||
| 4403 | static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS) { | ||||||||
| 4404 | switch (BinaryOp) { | ||||||||
| 4405 | default: | ||||||||
| 4406 | llvm_unreachable("Unsupported binary op")::llvm::llvm_unreachable_internal("Unsupported binary op", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4406); | ||||||||
| 4407 | case Instruction::Add: | ||||||||
| 4408 | case Instruction::Sub: | ||||||||
| 4409 | return match(RHS, m_Zero()); | ||||||||
| 4410 | case Instruction::Mul: | ||||||||
| 4411 | return match(RHS, m_One()); | ||||||||
| 4412 | } | ||||||||
| 4413 | } | ||||||||
| 4414 | |||||||||
| 4415 | OverflowResult InstCombiner::computeOverflow( | ||||||||
| 4416 | Instruction::BinaryOps BinaryOp, bool IsSigned, | ||||||||
| 4417 | Value *LHS, Value *RHS, Instruction *CxtI) const { | ||||||||
| 4418 | switch (BinaryOp) { | ||||||||
| 4419 | default: | ||||||||
| 4420 | llvm_unreachable("Unsupported binary op")::llvm::llvm_unreachable_internal("Unsupported binary op", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4420); | ||||||||
| 4421 | case Instruction::Add: | ||||||||
| 4422 | if (IsSigned) | ||||||||
| 4423 | return computeOverflowForSignedAdd(LHS, RHS, CxtI); | ||||||||
| 4424 | else | ||||||||
| 4425 | return computeOverflowForUnsignedAdd(LHS, RHS, CxtI); | ||||||||
| 4426 | case Instruction::Sub: | ||||||||
| 4427 | if (IsSigned) | ||||||||
| 4428 | return computeOverflowForSignedSub(LHS, RHS, CxtI); | ||||||||
| 4429 | else | ||||||||
| 4430 | return computeOverflowForUnsignedSub(LHS, RHS, CxtI); | ||||||||
| 4431 | case Instruction::Mul: | ||||||||
| 4432 | if (IsSigned) | ||||||||
| 4433 | return computeOverflowForSignedMul(LHS, RHS, CxtI); | ||||||||
| 4434 | else | ||||||||
| 4435 | return computeOverflowForUnsignedMul(LHS, RHS, CxtI); | ||||||||
| 4436 | } | ||||||||
| 4437 | } | ||||||||
| 4438 | |||||||||
| 4439 | bool InstCombiner::OptimizeOverflowCheck( | ||||||||
| 4440 | Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, | ||||||||
| 4441 | Instruction &OrigI, Value *&Result, Constant *&Overflow) { | ||||||||
| 4442 | if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS)) | ||||||||
| 4443 | std::swap(LHS, RHS); | ||||||||
| 4444 | |||||||||
| 4445 | // If the overflow check was an add followed by a compare, the insertion point | ||||||||
| 4446 | // may be pointing to the compare. We want to insert the new instructions | ||||||||
| 4447 | // before the add in case there are uses of the add between the add and the | ||||||||
| 4448 | // compare. | ||||||||
| 4449 | Builder.SetInsertPoint(&OrigI); | ||||||||
| 4450 | |||||||||
| 4451 | if (isNeutralValue(BinaryOp, RHS)) { | ||||||||
| 4452 | Result = LHS; | ||||||||
| 4453 | Overflow = Builder.getFalse(); | ||||||||
| 4454 | return true; | ||||||||
| 4455 | } | ||||||||
| 4456 | |||||||||
| 4457 | switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, &OrigI)) { | ||||||||
| 4458 | case OverflowResult::MayOverflow: | ||||||||
| 4459 | return false; | ||||||||
| 4460 | case OverflowResult::AlwaysOverflowsLow: | ||||||||
| 4461 | case OverflowResult::AlwaysOverflowsHigh: | ||||||||
| 4462 | Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); | ||||||||
| 4463 | Result->takeName(&OrigI); | ||||||||
| 4464 | Overflow = Builder.getTrue(); | ||||||||
| 4465 | return true; | ||||||||
| 4466 | case OverflowResult::NeverOverflows: | ||||||||
| 4467 | Result = Builder.CreateBinOp(BinaryOp, LHS, RHS); | ||||||||
| 4468 | Result->takeName(&OrigI); | ||||||||
| 4469 | Overflow = Builder.getFalse(); | ||||||||
| 4470 | if (auto *Inst = dyn_cast<Instruction>(Result)) { | ||||||||
| 4471 | if (IsSigned) | ||||||||
| 4472 | Inst->setHasNoSignedWrap(); | ||||||||
| 4473 | else | ||||||||
| 4474 | Inst->setHasNoUnsignedWrap(); | ||||||||
| 4475 | } | ||||||||
| 4476 | return true; | ||||||||
| 4477 | } | ||||||||
| 4478 | |||||||||
| 4479 | llvm_unreachable("Unexpected overflow result")::llvm::llvm_unreachable_internal("Unexpected overflow result" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4479); | ||||||||
| 4480 | } | ||||||||
| 4481 | |||||||||
| 4482 | /// Recognize and process idiom involving test for multiplication | ||||||||
| 4483 | /// overflow. | ||||||||
| 4484 | /// | ||||||||
| 4485 | /// The caller has matched a pattern of the form: | ||||||||
| 4486 | /// I = cmp u (mul(zext A, zext B), V | ||||||||
| 4487 | /// The function checks if this is a test for overflow and if so replaces | ||||||||
| 4488 | /// multiplication with call to 'mul.with.overflow' intrinsic. | ||||||||
| 4489 | /// | ||||||||
| 4490 | /// \param I Compare instruction. | ||||||||
| 4491 | /// \param MulVal Result of 'mult' instruction. It is one of the arguments of | ||||||||
| 4492 | /// the compare instruction. Must be of integer type. | ||||||||
| 4493 | /// \param OtherVal The other argument of compare instruction. | ||||||||
| 4494 | /// \returns Instruction which must replace the compare instruction, NULL if no | ||||||||
| 4495 | /// replacement required. | ||||||||
| 4496 | static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal, | ||||||||
| 4497 | Value *OtherVal, InstCombiner &IC) { | ||||||||
| 4498 | // Don't bother doing this transformation for pointers, don't do it for | ||||||||
| 4499 | // vectors. | ||||||||
| 4500 | if (!isa<IntegerType>(MulVal->getType())) | ||||||||
| 4501 | return nullptr; | ||||||||
| 4502 | |||||||||
| 4503 | assert(I.getOperand(0) == MulVal || I.getOperand(1) == MulVal)((I.getOperand(0) == MulVal || I.getOperand(1) == MulVal) ? static_cast <void> (0) : __assert_fail ("I.getOperand(0) == MulVal || I.getOperand(1) == MulVal" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4503, __PRETTY_FUNCTION__)); | ||||||||
| 4504 | assert(I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal)((I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal) ? static_cast<void> (0) : __assert_fail ("I.getOperand(0) == OtherVal || I.getOperand(1) == OtherVal" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4504, __PRETTY_FUNCTION__)); | ||||||||
| 4505 | auto *MulInstr = dyn_cast<Instruction>(MulVal); | ||||||||
| 4506 | if (!MulInstr) | ||||||||
| 4507 | return nullptr; | ||||||||
| 4508 | assert(MulInstr->getOpcode() == Instruction::Mul)((MulInstr->getOpcode() == Instruction::Mul) ? static_cast <void> (0) : __assert_fail ("MulInstr->getOpcode() == Instruction::Mul" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4508, __PRETTY_FUNCTION__)); | ||||||||
| 4509 | |||||||||
| 4510 | auto *LHS = cast<ZExtOperator>(MulInstr->getOperand(0)), | ||||||||
| 4511 | *RHS = cast<ZExtOperator>(MulInstr->getOperand(1)); | ||||||||
| 4512 | assert(LHS->getOpcode() == Instruction::ZExt)((LHS->getOpcode() == Instruction::ZExt) ? static_cast< void> (0) : __assert_fail ("LHS->getOpcode() == Instruction::ZExt" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4512, __PRETTY_FUNCTION__)); | ||||||||
| 4513 | assert(RHS->getOpcode() == Instruction::ZExt)((RHS->getOpcode() == Instruction::ZExt) ? static_cast< void> (0) : __assert_fail ("RHS->getOpcode() == Instruction::ZExt" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4513, __PRETTY_FUNCTION__)); | ||||||||
| 4514 | Value *A = LHS->getOperand(0), *B = RHS->getOperand(0); | ||||||||
| 4515 | |||||||||
| 4516 | // Calculate type and width of the result produced by mul.with.overflow. | ||||||||
| 4517 | Type *TyA = A->getType(), *TyB = B->getType(); | ||||||||
| 4518 | unsigned WidthA = TyA->getPrimitiveSizeInBits(), | ||||||||
| 4519 | WidthB = TyB->getPrimitiveSizeInBits(); | ||||||||
| 4520 | unsigned MulWidth; | ||||||||
| 4521 | Type *MulType; | ||||||||
| 4522 | if (WidthB > WidthA) { | ||||||||
| 4523 | MulWidth = WidthB; | ||||||||
| 4524 | MulType = TyB; | ||||||||
| 4525 | } else { | ||||||||
| 4526 | MulWidth = WidthA; | ||||||||
| 4527 | MulType = TyA; | ||||||||
| 4528 | } | ||||||||
| 4529 | |||||||||
| 4530 | // In order to replace the original mul with a narrower mul.with.overflow, | ||||||||
| 4531 | // all uses must ignore upper bits of the product. The number of used low | ||||||||
| 4532 | // bits must be not greater than the width of mul.with.overflow. | ||||||||
| 4533 | if (MulVal->hasNUsesOrMore(2)) | ||||||||
| 4534 | for (User *U : MulVal->users()) { | ||||||||
| 4535 | if (U == &I) | ||||||||
| 4536 | continue; | ||||||||
| 4537 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | ||||||||
| 4538 | // Check if truncation ignores bits above MulWidth. | ||||||||
| 4539 | unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits(); | ||||||||
| 4540 | if (TruncWidth > MulWidth) | ||||||||
| 4541 | return nullptr; | ||||||||
| 4542 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | ||||||||
| 4543 | // Check if AND ignores bits above MulWidth. | ||||||||
| 4544 | if (BO->getOpcode() != Instruction::And) | ||||||||
| 4545 | return nullptr; | ||||||||
| 4546 | if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1))) { | ||||||||
| 4547 | const APInt &CVal = CI->getValue(); | ||||||||
| 4548 | if (CVal.getBitWidth() - CVal.countLeadingZeros() > MulWidth) | ||||||||
| 4549 | return nullptr; | ||||||||
| 4550 | } else { | ||||||||
| 4551 | // In this case we could have the operand of the binary operation | ||||||||
| 4552 | // being defined in another block, and performing the replacement | ||||||||
| 4553 | // could break the dominance relation. | ||||||||
| 4554 | return nullptr; | ||||||||
| 4555 | } | ||||||||
| 4556 | } else { | ||||||||
| 4557 | // Other uses prohibit this transformation. | ||||||||
| 4558 | return nullptr; | ||||||||
| 4559 | } | ||||||||
| 4560 | } | ||||||||
| 4561 | |||||||||
| 4562 | // Recognize patterns | ||||||||
| 4563 | switch (I.getPredicate()) { | ||||||||
| 4564 | case ICmpInst::ICMP_EQ: | ||||||||
| 4565 | case ICmpInst::ICMP_NE: | ||||||||
| 4566 | // Recognize pattern: | ||||||||
| 4567 | // mulval = mul(zext A, zext B) | ||||||||
| 4568 | // cmp eq/neq mulval, zext trunc mulval | ||||||||
| 4569 | if (ZExtInst *Zext = dyn_cast<ZExtInst>(OtherVal)) | ||||||||
| 4570 | if (Zext->hasOneUse()) { | ||||||||
| 4571 | Value *ZextArg = Zext->getOperand(0); | ||||||||
| 4572 | if (TruncInst *Trunc = dyn_cast<TruncInst>(ZextArg)) | ||||||||
| 4573 | if (Trunc->getType()->getPrimitiveSizeInBits() == MulWidth) | ||||||||
| 4574 | break; //Recognized | ||||||||
| 4575 | } | ||||||||
| 4576 | |||||||||
| 4577 | // Recognize pattern: | ||||||||
| 4578 | // mulval = mul(zext A, zext B) | ||||||||
| 4579 | // cmp eq/neq mulval, and(mulval, mask), mask selects low MulWidth bits. | ||||||||
| 4580 | ConstantInt *CI; | ||||||||
| 4581 | Value *ValToMask; | ||||||||
| 4582 | if (match(OtherVal, m_And(m_Value(ValToMask), m_ConstantInt(CI)))) { | ||||||||
| 4583 | if (ValToMask != MulVal) | ||||||||
| 4584 | return nullptr; | ||||||||
| 4585 | const APInt &CVal = CI->getValue() + 1; | ||||||||
| 4586 | if (CVal.isPowerOf2()) { | ||||||||
| 4587 | unsigned MaskWidth = CVal.logBase2(); | ||||||||
| 4588 | if (MaskWidth == MulWidth) | ||||||||
| 4589 | break; // Recognized | ||||||||
| 4590 | } | ||||||||
| 4591 | } | ||||||||
| 4592 | return nullptr; | ||||||||
| 4593 | |||||||||
| 4594 | case ICmpInst::ICMP_UGT: | ||||||||
| 4595 | // Recognize pattern: | ||||||||
| 4596 | // mulval = mul(zext A, zext B) | ||||||||
| 4597 | // cmp ugt mulval, max | ||||||||
| 4598 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
| 4599 | APInt MaxVal = APInt::getMaxValue(MulWidth); | ||||||||
| 4600 | MaxVal = MaxVal.zext(CI->getBitWidth()); | ||||||||
| 4601 | if (MaxVal.eq(CI->getValue())) | ||||||||
| 4602 | break; // Recognized | ||||||||
| 4603 | } | ||||||||
| 4604 | return nullptr; | ||||||||
| 4605 | |||||||||
| 4606 | case ICmpInst::ICMP_UGE: | ||||||||
| 4607 | // Recognize pattern: | ||||||||
| 4608 | // mulval = mul(zext A, zext B) | ||||||||
| 4609 | // cmp uge mulval, max+1 | ||||||||
| 4610 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
| 4611 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | ||||||||
| 4612 | if (MaxVal.eq(CI->getValue())) | ||||||||
| 4613 | break; // Recognized | ||||||||
| 4614 | } | ||||||||
| 4615 | return nullptr; | ||||||||
| 4616 | |||||||||
| 4617 | case ICmpInst::ICMP_ULE: | ||||||||
| 4618 | // Recognize pattern: | ||||||||
| 4619 | // mulval = mul(zext A, zext B) | ||||||||
| 4620 | // cmp ule mulval, max | ||||||||
| 4621 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
| 4622 | APInt MaxVal = APInt::getMaxValue(MulWidth); | ||||||||
| 4623 | MaxVal = MaxVal.zext(CI->getBitWidth()); | ||||||||
| 4624 | if (MaxVal.eq(CI->getValue())) | ||||||||
| 4625 | break; // Recognized | ||||||||
| 4626 | } | ||||||||
| 4627 | return nullptr; | ||||||||
| 4628 | |||||||||
| 4629 | case ICmpInst::ICMP_ULT: | ||||||||
| 4630 | // Recognize pattern: | ||||||||
| 4631 | // mulval = mul(zext A, zext B) | ||||||||
| 4632 | // cmp ule mulval, max + 1 | ||||||||
| 4633 | if (ConstantInt *CI = dyn_cast<ConstantInt>(OtherVal)) { | ||||||||
| 4634 | APInt MaxVal = APInt::getOneBitSet(CI->getBitWidth(), MulWidth); | ||||||||
| 4635 | if (MaxVal.eq(CI->getValue())) | ||||||||
| 4636 | break; // Recognized | ||||||||
| 4637 | } | ||||||||
| 4638 | return nullptr; | ||||||||
| 4639 | |||||||||
| 4640 | default: | ||||||||
| 4641 | return nullptr; | ||||||||
| 4642 | } | ||||||||
| 4643 | |||||||||
| 4644 | InstCombiner::BuilderTy &Builder = IC.Builder; | ||||||||
| 4645 | Builder.SetInsertPoint(MulInstr); | ||||||||
| 4646 | |||||||||
| 4647 | // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B) | ||||||||
| 4648 | Value *MulA = A, *MulB = B; | ||||||||
| 4649 | if (WidthA < MulWidth) | ||||||||
| 4650 | MulA = Builder.CreateZExt(A, MulType); | ||||||||
| 4651 | if (WidthB < MulWidth) | ||||||||
| 4652 | MulB = Builder.CreateZExt(B, MulType); | ||||||||
| 4653 | Function *F = Intrinsic::getDeclaration( | ||||||||
| 4654 | I.getModule(), Intrinsic::umul_with_overflow, MulType); | ||||||||
| 4655 | CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul"); | ||||||||
| 4656 | IC.Worklist.push(MulInstr); | ||||||||
| 4657 | |||||||||
| 4658 | // If there are uses of mul result other than the comparison, we know that | ||||||||
| 4659 | // they are truncation or binary AND. Change them to use result of | ||||||||
| 4660 | // mul.with.overflow and adjust properly mask/size. | ||||||||
| 4661 | if (MulVal->hasNUsesOrMore(2)) { | ||||||||
| 4662 | Value *Mul = Builder.CreateExtractValue(Call, 0, "umul.value"); | ||||||||
| 4663 | for (auto UI = MulVal->user_begin(), UE = MulVal->user_end(); UI != UE;) { | ||||||||
| 4664 | User *U = *UI++; | ||||||||
| 4665 | if (U == &I || U == OtherVal) | ||||||||
| 4666 | continue; | ||||||||
| 4667 | if (TruncInst *TI = dyn_cast<TruncInst>(U)) { | ||||||||
| 4668 | if (TI->getType()->getPrimitiveSizeInBits() == MulWidth) | ||||||||
| 4669 | IC.replaceInstUsesWith(*TI, Mul); | ||||||||
| 4670 | else | ||||||||
| 4671 | TI->setOperand(0, Mul); | ||||||||
| 4672 | } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U)) { | ||||||||
| 4673 | assert(BO->getOpcode() == Instruction::And)((BO->getOpcode() == Instruction::And) ? static_cast<void > (0) : __assert_fail ("BO->getOpcode() == Instruction::And" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4673, __PRETTY_FUNCTION__)); | ||||||||
| 4674 | // Replace (mul & mask) --> zext (mul.with.overflow & short_mask) | ||||||||
| 4675 | ConstantInt *CI = cast<ConstantInt>(BO->getOperand(1)); | ||||||||
| 4676 | APInt ShortMask = CI->getValue().trunc(MulWidth); | ||||||||
| 4677 | Value *ShortAnd = Builder.CreateAnd(Mul, ShortMask); | ||||||||
| 4678 | Value *Zext = Builder.CreateZExt(ShortAnd, BO->getType()); | ||||||||
| 4679 | IC.replaceInstUsesWith(*BO, Zext); | ||||||||
| 4680 | } else { | ||||||||
| 4681 | llvm_unreachable("Unexpected Binary operation")::llvm::llvm_unreachable_internal("Unexpected Binary operation" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4681); | ||||||||
| 4682 | } | ||||||||
| 4683 | IC.Worklist.push(cast<Instruction>(U)); | ||||||||
| 4684 | } | ||||||||
| 4685 | } | ||||||||
| 4686 | if (isa<Instruction>(OtherVal)) | ||||||||
| 4687 | IC.Worklist.push(cast<Instruction>(OtherVal)); | ||||||||
| 4688 | |||||||||
| 4689 | // The original icmp gets replaced with the overflow value, maybe inverted | ||||||||
| 4690 | // depending on predicate. | ||||||||
| 4691 | bool Inverse = false; | ||||||||
| 4692 | switch (I.getPredicate()) { | ||||||||
| 4693 | case ICmpInst::ICMP_NE: | ||||||||
| 4694 | break; | ||||||||
| 4695 | case ICmpInst::ICMP_EQ: | ||||||||
| 4696 | Inverse = true; | ||||||||
| 4697 | break; | ||||||||
| 4698 | case ICmpInst::ICMP_UGT: | ||||||||
| 4699 | case ICmpInst::ICMP_UGE: | ||||||||
| 4700 | if (I.getOperand(0) == MulVal) | ||||||||
| 4701 | break; | ||||||||
| 4702 | Inverse = true; | ||||||||
| 4703 | break; | ||||||||
| 4704 | case ICmpInst::ICMP_ULT: | ||||||||
| 4705 | case ICmpInst::ICMP_ULE: | ||||||||
| 4706 | if (I.getOperand(1) == MulVal) | ||||||||
| 4707 | break; | ||||||||
| 4708 | Inverse = true; | ||||||||
| 4709 | break; | ||||||||
| 4710 | default: | ||||||||
| 4711 | llvm_unreachable("Unexpected predicate")::llvm::llvm_unreachable_internal("Unexpected predicate", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4711); | ||||||||
| 4712 | } | ||||||||
| 4713 | if (Inverse) { | ||||||||
| 4714 | Value *Res = Builder.CreateExtractValue(Call, 1); | ||||||||
| 4715 | return BinaryOperator::CreateNot(Res); | ||||||||
| 4716 | } | ||||||||
| 4717 | |||||||||
| 4718 | return ExtractValueInst::Create(Call, 1); | ||||||||
| 4719 | } | ||||||||
| 4720 | |||||||||
| 4721 | /// When performing a comparison against a constant, it is possible that not all | ||||||||
| 4722 | /// the bits in the LHS are demanded. This helper method computes the mask that | ||||||||
| 4723 | /// IS demanded. | ||||||||
| 4724 | static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) { | ||||||||
| 4725 | const APInt *RHS; | ||||||||
| 4726 | if (!match(I.getOperand(1), m_APInt(RHS))) | ||||||||
| 4727 | return APInt::getAllOnesValue(BitWidth); | ||||||||
| 4728 | |||||||||
| 4729 | // If this is a normal comparison, it demands all bits. If it is a sign bit | ||||||||
| 4730 | // comparison, it only demands the sign bit. | ||||||||
| 4731 | bool UnusedBit; | ||||||||
| 4732 | if (isSignBitCheck(I.getPredicate(), *RHS, UnusedBit)) | ||||||||
| 4733 | return APInt::getSignMask(BitWidth); | ||||||||
| 4734 | |||||||||
| 4735 | switch (I.getPredicate()) { | ||||||||
| 4736 | // For a UGT comparison, we don't care about any bits that | ||||||||
| 4737 | // correspond to the trailing ones of the comparand. The value of these | ||||||||
| 4738 | // bits doesn't impact the outcome of the comparison, because any value | ||||||||
| 4739 | // greater than the RHS must differ in a bit higher than these due to carry. | ||||||||
| 4740 | case ICmpInst::ICMP_UGT: | ||||||||
| 4741 | return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingOnes()); | ||||||||
| 4742 | |||||||||
| 4743 | // Similarly, for a ULT comparison, we don't care about the trailing zeros. | ||||||||
| 4744 | // Any value less than the RHS must differ in a higher bit because of carries. | ||||||||
| 4745 | case ICmpInst::ICMP_ULT: | ||||||||
| 4746 | return APInt::getBitsSetFrom(BitWidth, RHS->countTrailingZeros()); | ||||||||
| 4747 | |||||||||
| 4748 | default: | ||||||||
| 4749 | return APInt::getAllOnesValue(BitWidth); | ||||||||
| 4750 | } | ||||||||
| 4751 | } | ||||||||
| 4752 | |||||||||
| 4753 | /// Check if the order of \p Op0 and \p Op1 as operands in an ICmpInst | ||||||||
| 4754 | /// should be swapped. | ||||||||
| 4755 | /// The decision is based on how many times these two operands are reused | ||||||||
| 4756 | /// as subtract operands and their positions in those instructions. | ||||||||
| 4757 | /// The rationale is that several architectures use the same instruction for | ||||||||
| 4758 | /// both subtract and cmp. Thus, it is better if the order of those operands | ||||||||
| 4759 | /// match. | ||||||||
| 4760 | /// \return true if Op0 and Op1 should be swapped. | ||||||||
| 4761 | static bool swapMayExposeCSEOpportunities(const Value *Op0, const Value *Op1) { | ||||||||
| 4762 | // Filter out pointer values as those cannot appear directly in subtract. | ||||||||
| 4763 | // FIXME: we may want to go through inttoptrs or bitcasts. | ||||||||
| 4764 | if (Op0->getType()->isPointerTy()) | ||||||||
| 4765 | return false; | ||||||||
| 4766 | // If a subtract already has the same operands as a compare, swapping would be | ||||||||
| 4767 | // bad. If a subtract has the same operands as a compare but in reverse order, | ||||||||
| 4768 | // then swapping is good. | ||||||||
| 4769 | int GoodToSwap = 0; | ||||||||
| 4770 | for (const User *U : Op0->users()) { | ||||||||
| 4771 | if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0)))) | ||||||||
| 4772 | GoodToSwap++; | ||||||||
| 4773 | else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1)))) | ||||||||
| 4774 | GoodToSwap--; | ||||||||
| 4775 | } | ||||||||
| 4776 | return GoodToSwap > 0; | ||||||||
| 4777 | } | ||||||||
| 4778 | |||||||||
| 4779 | /// Check that one use is in the same block as the definition and all | ||||||||
| 4780 | /// other uses are in blocks dominated by a given block. | ||||||||
| 4781 | /// | ||||||||
| 4782 | /// \param DI Definition | ||||||||
| 4783 | /// \param UI Use | ||||||||
| 4784 | /// \param DB Block that must dominate all uses of \p DI outside | ||||||||
| 4785 | /// the parent block | ||||||||
| 4786 | /// \return true when \p UI is the only use of \p DI in the parent block | ||||||||
| 4787 | /// and all other uses of \p DI are in blocks dominated by \p DB. | ||||||||
| 4788 | /// | ||||||||
| 4789 | bool InstCombiner::dominatesAllUses(const Instruction *DI, | ||||||||
| 4790 | const Instruction *UI, | ||||||||
| 4791 | const BasicBlock *DB) const { | ||||||||
| 4792 | assert(DI && UI && "Instruction not defined\n")((DI && UI && "Instruction not defined\n") ? static_cast <void> (0) : __assert_fail ("DI && UI && \"Instruction not defined\\n\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4792, __PRETTY_FUNCTION__)); | ||||||||
| 4793 | // Ignore incomplete definitions. | ||||||||
| 4794 | if (!DI->getParent()) | ||||||||
| 4795 | return false; | ||||||||
| 4796 | // DI and UI must be in the same block. | ||||||||
| 4797 | if (DI->getParent() != UI->getParent()) | ||||||||
| 4798 | return false; | ||||||||
| 4799 | // Protect from self-referencing blocks. | ||||||||
| 4800 | if (DI->getParent() == DB) | ||||||||
| 4801 | return false; | ||||||||
| 4802 | for (const User *U : DI->users()) { | ||||||||
| 4803 | auto *Usr = cast<Instruction>(U); | ||||||||
| 4804 | if (Usr != UI && !DT.dominates(DB, Usr->getParent())) | ||||||||
| 4805 | return false; | ||||||||
| 4806 | } | ||||||||
| 4807 | return true; | ||||||||
| 4808 | } | ||||||||
| 4809 | |||||||||
| 4810 | /// Return true when the instruction sequence within a block is select-cmp-br. | ||||||||
| 4811 | static bool isChainSelectCmpBranch(const SelectInst *SI) { | ||||||||
| 4812 | const BasicBlock *BB = SI->getParent(); | ||||||||
| 4813 | if (!BB) | ||||||||
| 4814 | return false; | ||||||||
| 4815 | auto *BI = dyn_cast_or_null<BranchInst>(BB->getTerminator()); | ||||||||
| 4816 | if (!BI || BI->getNumSuccessors() != 2) | ||||||||
| 4817 | return false; | ||||||||
| 4818 | auto *IC = dyn_cast<ICmpInst>(BI->getCondition()); | ||||||||
| 4819 | if (!IC || (IC->getOperand(0) != SI && IC->getOperand(1) != SI)) | ||||||||
| 4820 | return false; | ||||||||
| 4821 | return true; | ||||||||
| 4822 | } | ||||||||
| 4823 | |||||||||
| 4824 | /// True when a select result is replaced by one of its operands | ||||||||
| 4825 | /// in select-icmp sequence. This will eventually result in the elimination | ||||||||
| 4826 | /// of the select. | ||||||||
| 4827 | /// | ||||||||
| 4828 | /// \param SI Select instruction | ||||||||
| 4829 | /// \param Icmp Compare instruction | ||||||||
| 4830 | /// \param SIOpd Operand that replaces the select | ||||||||
| 4831 | /// | ||||||||
| 4832 | /// Notes: | ||||||||
| 4833 | /// - The replacement is global and requires dominator information | ||||||||
| 4834 | /// - The caller is responsible for the actual replacement | ||||||||
| 4835 | /// | ||||||||
| 4836 | /// Example: | ||||||||
| 4837 | /// | ||||||||
| 4838 | /// entry: | ||||||||
| 4839 | /// %4 = select i1 %3, %C* %0, %C* null | ||||||||
| 4840 | /// %5 = icmp eq %C* %4, null | ||||||||
| 4841 | /// br i1 %5, label %9, label %7 | ||||||||
| 4842 | /// ... | ||||||||
| 4843 | /// ; <label>:7 ; preds = %entry | ||||||||
| 4844 | /// %8 = getelementptr inbounds %C* %4, i64 0, i32 0 | ||||||||
| 4845 | /// ... | ||||||||
| 4846 | /// | ||||||||
| 4847 | /// can be transformed to | ||||||||
| 4848 | /// | ||||||||
| 4849 | /// %5 = icmp eq %C* %0, null | ||||||||
| 4850 | /// %6 = select i1 %3, i1 %5, i1 true | ||||||||
| 4851 | /// br i1 %6, label %9, label %7 | ||||||||
| 4852 | /// ... | ||||||||
| 4853 | /// ; <label>:7 ; preds = %entry | ||||||||
| 4854 | /// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0! | ||||||||
| 4855 | /// | ||||||||
| 4856 | /// Similar when the first operand of the select is a constant or/and | ||||||||
| 4857 | /// the compare is for not equal rather than equal. | ||||||||
| 4858 | /// | ||||||||
| 4859 | /// NOTE: The function is only called when the select and compare constants | ||||||||
| 4860 | /// are equal, the optimization can work only for EQ predicates. This is not a | ||||||||
| 4861 | /// major restriction since a NE compare should be 'normalized' to an equal | ||||||||
| 4862 | /// compare, which usually happens in the combiner and test case | ||||||||
| 4863 | /// select-cmp-br.ll checks for it. | ||||||||
| 4864 | bool InstCombiner::replacedSelectWithOperand(SelectInst *SI, | ||||||||
| 4865 | const ICmpInst *Icmp, | ||||||||
| 4866 | const unsigned SIOpd) { | ||||||||
| 4867 | assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!")(((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!" ) ? static_cast<void> (0) : __assert_fail ("(SIOpd == 1 || SIOpd == 2) && \"Invalid select operand!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4867, __PRETTY_FUNCTION__)); | ||||||||
| 4868 | if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) { | ||||||||
| 4869 | BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1); | ||||||||
| 4870 | // The check for the single predecessor is not the best that can be | ||||||||
| 4871 | // done. But it protects efficiently against cases like when SI's | ||||||||
| 4872 | // home block has two successors, Succ and Succ1, and Succ1 predecessor | ||||||||
| 4873 | // of Succ. Then SI can't be replaced by SIOpd because the use that gets | ||||||||
| 4874 | // replaced can be reached on either path. So the uniqueness check | ||||||||
| 4875 | // guarantees that the path all uses of SI (outside SI's parent) are on | ||||||||
| 4876 | // is disjoint from all other paths out of SI. But that information | ||||||||
| 4877 | // is more expensive to compute, and the trade-off here is in favor | ||||||||
| 4878 | // of compile-time. It should also be noticed that we check for a single | ||||||||
| 4879 | // predecessor and not only uniqueness. This to handle the situation when | ||||||||
| 4880 | // Succ and Succ1 points to the same basic block. | ||||||||
| 4881 | if (Succ->getSinglePredecessor() && dominatesAllUses(SI, Icmp, Succ)) { | ||||||||
| 4882 | NumSel++; | ||||||||
| 4883 | SI->replaceUsesOutsideBlock(SI->getOperand(SIOpd), SI->getParent()); | ||||||||
| 4884 | return true; | ||||||||
| 4885 | } | ||||||||
| 4886 | } | ||||||||
| 4887 | return false; | ||||||||
| 4888 | } | ||||||||
| 4889 | |||||||||
| 4890 | /// Try to fold the comparison based on range information we can get by checking | ||||||||
| 4891 | /// whether bits are known to be zero or one in the inputs. | ||||||||
| 4892 | Instruction *InstCombiner::foldICmpUsingKnownBits(ICmpInst &I) { | ||||||||
| 4893 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 4894 | Type *Ty = Op0->getType(); | ||||||||
| 4895 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 4896 | |||||||||
| 4897 | // Get scalar or pointer size. | ||||||||
| 4898 | unsigned BitWidth = Ty->isIntOrIntVectorTy() | ||||||||
| 4899 | ? Ty->getScalarSizeInBits() | ||||||||
| 4900 | : DL.getPointerTypeSizeInBits(Ty->getScalarType()); | ||||||||
| 4901 | |||||||||
| 4902 | if (!BitWidth) | ||||||||
| 4903 | return nullptr; | ||||||||
| 4904 | |||||||||
| 4905 | KnownBits Op0Known(BitWidth); | ||||||||
| 4906 | KnownBits Op1Known(BitWidth); | ||||||||
| 4907 | |||||||||
| 4908 | if (SimplifyDemandedBits(&I, 0, | ||||||||
| 4909 | getDemandedBitsLHSMask(I, BitWidth), | ||||||||
| 4910 | Op0Known, 0)) | ||||||||
| 4911 | return &I; | ||||||||
| 4912 | |||||||||
| 4913 | if (SimplifyDemandedBits(&I, 1, APInt::getAllOnesValue(BitWidth), | ||||||||
| 4914 | Op1Known, 0)) | ||||||||
| 4915 | return &I; | ||||||||
| 4916 | |||||||||
| 4917 | // Given the known and unknown bits, compute a range that the LHS could be | ||||||||
| 4918 | // in. Compute the Min, Max and RHS values based on the known bits. For the | ||||||||
| 4919 | // EQ and NE we use unsigned values. | ||||||||
| 4920 | APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0); | ||||||||
| 4921 | APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0); | ||||||||
| 4922 | if (I.isSigned()) { | ||||||||
| 4923 | computeSignedMinMaxValuesFromKnownBits(Op0Known, Op0Min, Op0Max); | ||||||||
| 4924 | computeSignedMinMaxValuesFromKnownBits(Op1Known, Op1Min, Op1Max); | ||||||||
| 4925 | } else { | ||||||||
| 4926 | computeUnsignedMinMaxValuesFromKnownBits(Op0Known, Op0Min, Op0Max); | ||||||||
| 4927 | computeUnsignedMinMaxValuesFromKnownBits(Op1Known, Op1Min, Op1Max); | ||||||||
| 4928 | } | ||||||||
| 4929 | |||||||||
| 4930 | // If Min and Max are known to be the same, then SimplifyDemandedBits figured | ||||||||
| 4931 | // out that the LHS or RHS is a constant. Constant fold this now, so that | ||||||||
| 4932 | // code below can assume that Min != Max. | ||||||||
| 4933 | if (!isa<Constant>(Op0) && Op0Min == Op0Max) | ||||||||
| 4934 | return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, Op0Min), Op1); | ||||||||
| 4935 | if (!isa<Constant>(Op1) && Op1Min == Op1Max) | ||||||||
| 4936 | return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, Op1Min)); | ||||||||
| 4937 | |||||||||
| 4938 | // Based on the range information we know about the LHS, see if we can | ||||||||
| 4939 | // simplify this comparison. For example, (x&4) < 8 is always true. | ||||||||
| 4940 | switch (Pred) { | ||||||||
| 4941 | default: | ||||||||
| 4942 | llvm_unreachable("Unknown icmp opcode!")::llvm::llvm_unreachable_internal("Unknown icmp opcode!", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 4942); | ||||||||
| 4943 | case ICmpInst::ICMP_EQ: | ||||||||
| 4944 | case ICmpInst::ICMP_NE: { | ||||||||
| 4945 | if (Op0Max.ult(Op1Min) || Op0Min.ugt(Op1Max)) { | ||||||||
| 4946 | return Pred == CmpInst::ICMP_EQ | ||||||||
| 4947 | ? replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())) | ||||||||
| 4948 | : replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 4949 | } | ||||||||
| 4950 | |||||||||
| 4951 | // If all bits are known zero except for one, then we know at most one bit | ||||||||
| 4952 | // is set. If the comparison is against zero, then this is a check to see if | ||||||||
| 4953 | // *that* bit is set. | ||||||||
| 4954 | APInt Op0KnownZeroInverted = ~Op0Known.Zero; | ||||||||
| 4955 | if (Op1Known.isZero()) { | ||||||||
| 4956 | // If the LHS is an AND with the same constant, look through it. | ||||||||
| 4957 | Value *LHS = nullptr; | ||||||||
| 4958 | const APInt *LHSC; | ||||||||
| 4959 | if (!match(Op0, m_And(m_Value(LHS), m_APInt(LHSC))) || | ||||||||
| 4960 | *LHSC != Op0KnownZeroInverted) | ||||||||
| 4961 | LHS = Op0; | ||||||||
| 4962 | |||||||||
| 4963 | Value *X; | ||||||||
| 4964 | if (match(LHS, m_Shl(m_One(), m_Value(X)))) { | ||||||||
| 4965 | APInt ValToCheck = Op0KnownZeroInverted; | ||||||||
| 4966 | Type *XTy = X->getType(); | ||||||||
| 4967 | if (ValToCheck.isPowerOf2()) { | ||||||||
| 4968 | // ((1 << X) & 8) == 0 -> X != 3 | ||||||||
| 4969 | // ((1 << X) & 8) != 0 -> X == 3 | ||||||||
| 4970 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | ||||||||
| 4971 | auto NewPred = ICmpInst::getInversePredicate(Pred); | ||||||||
| 4972 | return new ICmpInst(NewPred, X, CmpC); | ||||||||
| 4973 | } else if ((++ValToCheck).isPowerOf2()) { | ||||||||
| 4974 | // ((1 << X) & 7) == 0 -> X >= 3 | ||||||||
| 4975 | // ((1 << X) & 7) != 0 -> X < 3 | ||||||||
| 4976 | auto *CmpC = ConstantInt::get(XTy, ValToCheck.countTrailingZeros()); | ||||||||
| 4977 | auto NewPred = | ||||||||
| 4978 | Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT; | ||||||||
| 4979 | return new ICmpInst(NewPred, X, CmpC); | ||||||||
| 4980 | } | ||||||||
| 4981 | } | ||||||||
| 4982 | |||||||||
| 4983 | // Check if the LHS is 8 >>u x and the result is a power of 2 like 1. | ||||||||
| 4984 | const APInt *CI; | ||||||||
| 4985 | if (Op0KnownZeroInverted.isOneValue() && | ||||||||
| 4986 | match(LHS, m_LShr(m_Power2(CI), m_Value(X)))) { | ||||||||
| 4987 | // ((8 >>u X) & 1) == 0 -> X != 3 | ||||||||
| 4988 | // ((8 >>u X) & 1) != 0 -> X == 3 | ||||||||
| 4989 | unsigned CmpVal = CI->countTrailingZeros(); | ||||||||
| 4990 | auto NewPred = ICmpInst::getInversePredicate(Pred); | ||||||||
| 4991 | return new ICmpInst(NewPred, X, ConstantInt::get(X->getType(), CmpVal)); | ||||||||
| 4992 | } | ||||||||
| 4993 | } | ||||||||
| 4994 | break; | ||||||||
| 4995 | } | ||||||||
| 4996 | case ICmpInst::ICMP_ULT: { | ||||||||
| 4997 | if (Op0Max.ult(Op1Min)) // A <u B -> true if max(A) < min(B) | ||||||||
| 4998 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 4999 | if (Op0Min.uge(Op1Max)) // A <u B -> false if min(A) >= max(B) | ||||||||
| 5000 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5001 | if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B) | ||||||||
| 5002 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
| 5003 | |||||||||
| 5004 | const APInt *CmpC; | ||||||||
| 5005 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
| 5006 | // A <u C -> A == C-1 if min(A)+1 == C | ||||||||
| 5007 | if (*CmpC == Op0Min + 1) | ||||||||
| 5008 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
| 5009 | ConstantInt::get(Op1->getType(), *CmpC - 1)); | ||||||||
| 5010 | // X <u C --> X == 0, if the number of zero bits in the bottom of X | ||||||||
| 5011 | // exceeds the log2 of C. | ||||||||
| 5012 | if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2()) | ||||||||
| 5013 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
| 5014 | Constant::getNullValue(Op1->getType())); | ||||||||
| 5015 | } | ||||||||
| 5016 | break; | ||||||||
| 5017 | } | ||||||||
| 5018 | case ICmpInst::ICMP_UGT: { | ||||||||
| 5019 | if (Op0Min.ugt(Op1Max)) // A >u B -> true if min(A) > max(B) | ||||||||
| 5020 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 5021 | if (Op0Max.ule(Op1Min)) // A >u B -> false if max(A) <= max(B) | ||||||||
| 5022 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5023 | if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B) | ||||||||
| 5024 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
| 5025 | |||||||||
| 5026 | const APInt *CmpC; | ||||||||
| 5027 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
| 5028 | // A >u C -> A == C+1 if max(a)-1 == C | ||||||||
| 5029 | if (*CmpC == Op0Max - 1) | ||||||||
| 5030 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
| 5031 | ConstantInt::get(Op1->getType(), *CmpC + 1)); | ||||||||
| 5032 | // X >u C --> X != 0, if the number of zero bits in the bottom of X | ||||||||
| 5033 | // exceeds the log2 of C. | ||||||||
| 5034 | if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits()) | ||||||||
| 5035 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, | ||||||||
| 5036 | Constant::getNullValue(Op1->getType())); | ||||||||
| 5037 | } | ||||||||
| 5038 | break; | ||||||||
| 5039 | } | ||||||||
| 5040 | case ICmpInst::ICMP_SLT: { | ||||||||
| 5041 | if (Op0Max.slt(Op1Min)) // A <s B -> true if max(A) < min(C) | ||||||||
| 5042 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 5043 | if (Op0Min.sge(Op1Max)) // A <s B -> false if min(A) >= max(C) | ||||||||
| 5044 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5045 | if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B) | ||||||||
| 5046 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
| 5047 | const APInt *CmpC; | ||||||||
| 5048 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
| 5049 | if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C | ||||||||
| 5050 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
| 5051 | ConstantInt::get(Op1->getType(), *CmpC - 1)); | ||||||||
| 5052 | } | ||||||||
| 5053 | break; | ||||||||
| 5054 | } | ||||||||
| 5055 | case ICmpInst::ICMP_SGT: { | ||||||||
| 5056 | if (Op0Min.sgt(Op1Max)) // A >s B -> true if min(A) > max(B) | ||||||||
| 5057 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 5058 | if (Op0Max.sle(Op1Min)) // A >s B -> false if max(A) <= min(B) | ||||||||
| 5059 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5060 | if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B) | ||||||||
| 5061 | return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1); | ||||||||
| 5062 | const APInt *CmpC; | ||||||||
| 5063 | if (match(Op1, m_APInt(CmpC))) { | ||||||||
| 5064 | if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C | ||||||||
| 5065 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, | ||||||||
| 5066 | ConstantInt::get(Op1->getType(), *CmpC + 1)); | ||||||||
| 5067 | } | ||||||||
| 5068 | break; | ||||||||
| 5069 | } | ||||||||
| 5070 | case ICmpInst::ICMP_SGE: | ||||||||
| 5071 | assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SGE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5071, __PRETTY_FUNCTION__)); | ||||||||
| 5072 | if (Op0Min.sge(Op1Max)) // A >=s B -> true if min(A) >= max(B) | ||||||||
| 5073 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 5074 | if (Op0Max.slt(Op1Min)) // A >=s B -> false if max(A) < min(B) | ||||||||
| 5075 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5076 | if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B) | ||||||||
| 5077 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
| 5078 | break; | ||||||||
| 5079 | case ICmpInst::ICMP_SLE: | ||||||||
| 5080 | assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_SLE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5080, __PRETTY_FUNCTION__)); | ||||||||
| 5081 | if (Op0Max.sle(Op1Min)) // A <=s B -> true if max(A) <= min(B) | ||||||||
| 5082 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 5083 | if (Op0Min.sgt(Op1Max)) // A <=s B -> false if min(A) > max(B) | ||||||||
| 5084 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5085 | if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B) | ||||||||
| 5086 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
| 5087 | break; | ||||||||
| 5088 | case ICmpInst::ICMP_UGE: | ||||||||
| 5089 | assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_UGE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5089, __PRETTY_FUNCTION__)); | ||||||||
| 5090 | if (Op0Min.uge(Op1Max)) // A >=u B -> true if min(A) >= max(B) | ||||||||
| 5091 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 5092 | if (Op0Max.ult(Op1Min)) // A >=u B -> false if max(A) < min(B) | ||||||||
| 5093 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5094 | if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B) | ||||||||
| 5095 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
| 5096 | break; | ||||||||
| 5097 | case ICmpInst::ICMP_ULE: | ||||||||
| 5098 | assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!")((!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!" ) ? static_cast<void> (0) : __assert_fail ("!isa<ConstantInt>(Op1) && \"ICMP_ULE with ConstantInt not folded!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5098, __PRETTY_FUNCTION__)); | ||||||||
| 5099 | if (Op0Max.ule(Op1Min)) // A <=u B -> true if max(A) <= min(B) | ||||||||
| 5100 | return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); | ||||||||
| 5101 | if (Op0Min.ugt(Op1Max)) // A <=u B -> false if min(A) > max(B) | ||||||||
| 5102 | return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); | ||||||||
| 5103 | if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B) | ||||||||
| 5104 | return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1); | ||||||||
| 5105 | break; | ||||||||
| 5106 | } | ||||||||
| 5107 | |||||||||
| 5108 | // Turn a signed comparison into an unsigned one if both operands are known to | ||||||||
| 5109 | // have the same sign. | ||||||||
| 5110 | if (I.isSigned() && | ||||||||
| 5111 | ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) || | ||||||||
| 5112 | (Op0Known.One.isNegative() && Op1Known.One.isNegative()))) | ||||||||
| 5113 | return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1); | ||||||||
| 5114 | |||||||||
| 5115 | return nullptr; | ||||||||
| 5116 | } | ||||||||
| 5117 | |||||||||
| 5118 | llvm::Optional<std::pair<CmpInst::Predicate, Constant *>> | ||||||||
| 5119 | llvm::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, | ||||||||
| 5120 | Constant *C) { | ||||||||
| 5121 | assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&((ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate (Pred) && "Only for relational integer predicates.") ? static_cast<void> (0) : __assert_fail ("ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && \"Only for relational integer predicates.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5122, __PRETTY_FUNCTION__)) | ||||||||
| 5122 | "Only for relational integer predicates.")((ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate (Pred) && "Only for relational integer predicates.") ? static_cast<void> (0) : __assert_fail ("ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) && \"Only for relational integer predicates.\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5122, __PRETTY_FUNCTION__)); | ||||||||
| 5123 | |||||||||
| 5124 | Type *Type = C->getType(); | ||||||||
| 5125 | bool IsSigned = ICmpInst::isSigned(Pred); | ||||||||
| 5126 | |||||||||
| 5127 | CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred); | ||||||||
| 5128 | bool WillIncrement = | ||||||||
| 5129 | UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT; | ||||||||
| 5130 | |||||||||
| 5131 | // Check if the constant operand can be safely incremented/decremented | ||||||||
| 5132 | // without overflowing/underflowing. | ||||||||
| 5133 | auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) { | ||||||||
| 5134 | return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned); | ||||||||
| 5135 | }; | ||||||||
| 5136 | |||||||||
| 5137 | Constant *SafeReplacementConstant = nullptr; | ||||||||
| 5138 | if (auto *CI = dyn_cast<ConstantInt>(C)) { | ||||||||
| 5139 | // Bail out if the constant can't be safely incremented/decremented. | ||||||||
| 5140 | if (!ConstantIsOk(CI)) | ||||||||
| 5141 | return llvm::None; | ||||||||
| 5142 | } else if (Type->isVectorTy()) { | ||||||||
| 5143 | unsigned NumElts = Type->getVectorNumElements(); | ||||||||
| 5144 | for (unsigned i = 0; i != NumElts; ++i) { | ||||||||
| 5145 | Constant *Elt = C->getAggregateElement(i); | ||||||||
| 5146 | if (!Elt) | ||||||||
| 5147 | return llvm::None; | ||||||||
| 5148 | |||||||||
| 5149 | if (isa<UndefValue>(Elt)) | ||||||||
| 5150 | continue; | ||||||||
| 5151 | |||||||||
| 5152 | // Bail out if we can't determine if this constant is min/max or if we | ||||||||
| 5153 | // know that this constant is min/max. | ||||||||
| 5154 | auto *CI = dyn_cast<ConstantInt>(Elt); | ||||||||
| 5155 | if (!CI || !ConstantIsOk(CI)) | ||||||||
| 5156 | return llvm::None; | ||||||||
| 5157 | |||||||||
| 5158 | if (!SafeReplacementConstant) | ||||||||
| 5159 | SafeReplacementConstant = CI; | ||||||||
| 5160 | } | ||||||||
| 5161 | } else { | ||||||||
| 5162 | // ConstantExpr? | ||||||||
| 5163 | return llvm::None; | ||||||||
| 5164 | } | ||||||||
| 5165 | |||||||||
| 5166 | // It may not be safe to change a compare predicate in the presence of | ||||||||
| 5167 | // undefined elements, so replace those elements with the first safe constant | ||||||||
| 5168 | // that we found. | ||||||||
| 5169 | if (C->containsUndefElement()) { | ||||||||
| 5170 | assert(SafeReplacementConstant && "Replacement constant not set")((SafeReplacementConstant && "Replacement constant not set" ) ? static_cast<void> (0) : __assert_fail ("SafeReplacementConstant && \"Replacement constant not set\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5170, __PRETTY_FUNCTION__)); | ||||||||
| 5171 | C = Constant::replaceUndefsWith(C, SafeReplacementConstant); | ||||||||
| 5172 | } | ||||||||
| 5173 | |||||||||
| 5174 | CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(Pred); | ||||||||
| 5175 | |||||||||
| 5176 | // Increment or decrement the constant. | ||||||||
| 5177 | Constant *OneOrNegOne = ConstantInt::get(Type, WillIncrement ? 1 : -1, true); | ||||||||
| 5178 | Constant *NewC = ConstantExpr::getAdd(C, OneOrNegOne); | ||||||||
| 5179 | |||||||||
| 5180 | return std::make_pair(NewPred, NewC); | ||||||||
| 5181 | } | ||||||||
| 5182 | |||||||||
| 5183 | /// If we have an icmp le or icmp ge instruction with a constant operand, turn | ||||||||
| 5184 | /// it into the appropriate icmp lt or icmp gt instruction. This transform | ||||||||
| 5185 | /// allows them to be folded in visitICmpInst. | ||||||||
| 5186 | static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) { | ||||||||
| 5187 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 5188 | if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) || | ||||||||
| 5189 | isCanonicalPredicate(Pred)) | ||||||||
| 5190 | return nullptr; | ||||||||
| 5191 | |||||||||
| 5192 | Value *Op0 = I.getOperand(0); | ||||||||
| 5193 | Value *Op1 = I.getOperand(1); | ||||||||
| 5194 | auto *Op1C = dyn_cast<Constant>(Op1); | ||||||||
| 5195 | if (!Op1C) | ||||||||
| 5196 | return nullptr; | ||||||||
| 5197 | |||||||||
| 5198 | auto FlippedStrictness = getFlippedStrictnessPredicateAndConstant(Pred, Op1C); | ||||||||
| 5199 | if (!FlippedStrictness) | ||||||||
| 5200 | return nullptr; | ||||||||
| 5201 | |||||||||
| 5202 | return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second); | ||||||||
| 5203 | } | ||||||||
| 5204 | |||||||||
| 5205 | /// Integer compare with boolean values can always be turned into bitwise ops. | ||||||||
| 5206 | static Instruction *canonicalizeICmpBool(ICmpInst &I, | ||||||||
| 5207 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 5208 | Value *A = I.getOperand(0), *B = I.getOperand(1); | ||||||||
| 5209 | assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only")((A->getType()->isIntOrIntVectorTy(1) && "Bools only" ) ? static_cast<void> (0) : __assert_fail ("A->getType()->isIntOrIntVectorTy(1) && \"Bools only\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5209, __PRETTY_FUNCTION__)); | ||||||||
| 5210 | |||||||||
| 5211 | // A boolean compared to true/false can be simplified to Op0/true/false in | ||||||||
| 5212 | // 14 out of the 20 (10 predicates * 2 constants) possible combinations. | ||||||||
| 5213 | // Cases not handled by InstSimplify are always 'not' of Op0. | ||||||||
| 5214 | if (match(B, m_Zero())) { | ||||||||
| 5215 | switch (I.getPredicate()) { | ||||||||
| 5216 | case CmpInst::ICMP_EQ: // A == 0 -> !A | ||||||||
| 5217 | case CmpInst::ICMP_ULE: // A <=u 0 -> !A | ||||||||
| 5218 | case CmpInst::ICMP_SGE: // A >=s 0 -> !A | ||||||||
| 5219 | return BinaryOperator::CreateNot(A); | ||||||||
| 5220 | default: | ||||||||
| 5221 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5221); | ||||||||
| 5222 | } | ||||||||
| 5223 | } else if (match(B, m_One())) { | ||||||||
| 5224 | switch (I.getPredicate()) { | ||||||||
| 5225 | case CmpInst::ICMP_NE: // A != 1 -> !A | ||||||||
| 5226 | case CmpInst::ICMP_ULT: // A <u 1 -> !A | ||||||||
| 5227 | case CmpInst::ICMP_SGT: // A >s -1 -> !A | ||||||||
| 5228 | return BinaryOperator::CreateNot(A); | ||||||||
| 5229 | default: | ||||||||
| 5230 | llvm_unreachable("ICmp i1 X, C not simplified as expected.")::llvm::llvm_unreachable_internal("ICmp i1 X, C not simplified as expected." , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5230); | ||||||||
| 5231 | } | ||||||||
| 5232 | } | ||||||||
| 5233 | |||||||||
| 5234 | switch (I.getPredicate()) { | ||||||||
| 5235 | default: | ||||||||
| 5236 | llvm_unreachable("Invalid icmp instruction!")::llvm::llvm_unreachable_internal("Invalid icmp instruction!" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5236); | ||||||||
| 5237 | case ICmpInst::ICMP_EQ: | ||||||||
| 5238 | // icmp eq i1 A, B -> ~(A ^ B) | ||||||||
| 5239 | return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); | ||||||||
| 5240 | |||||||||
| 5241 | case ICmpInst::ICMP_NE: | ||||||||
| 5242 | // icmp ne i1 A, B -> A ^ B | ||||||||
| 5243 | return BinaryOperator::CreateXor(A, B); | ||||||||
| 5244 | |||||||||
| 5245 | case ICmpInst::ICMP_UGT: | ||||||||
| 5246 | // icmp ugt -> icmp ult | ||||||||
| 5247 | std::swap(A, B); | ||||||||
| 5248 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
| 5249 | case ICmpInst::ICMP_ULT: | ||||||||
| 5250 | // icmp ult i1 A, B -> ~A & B | ||||||||
| 5251 | return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); | ||||||||
| 5252 | |||||||||
| 5253 | case ICmpInst::ICMP_SGT: | ||||||||
| 5254 | // icmp sgt -> icmp slt | ||||||||
| 5255 | std::swap(A, B); | ||||||||
| 5256 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
| 5257 | case ICmpInst::ICMP_SLT: | ||||||||
| 5258 | // icmp slt i1 A, B -> A & ~B | ||||||||
| 5259 | return BinaryOperator::CreateAnd(Builder.CreateNot(B), A); | ||||||||
| 5260 | |||||||||
| 5261 | case ICmpInst::ICMP_UGE: | ||||||||
| 5262 | // icmp uge -> icmp ule | ||||||||
| 5263 | std::swap(A, B); | ||||||||
| 5264 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
| 5265 | case ICmpInst::ICMP_ULE: | ||||||||
| 5266 | // icmp ule i1 A, B -> ~A | B | ||||||||
| 5267 | return BinaryOperator::CreateOr(Builder.CreateNot(A), B); | ||||||||
| 5268 | |||||||||
| 5269 | case ICmpInst::ICMP_SGE: | ||||||||
| 5270 | // icmp sge -> icmp sle | ||||||||
| 5271 | std::swap(A, B); | ||||||||
| 5272 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||||
| 5273 | case ICmpInst::ICMP_SLE: | ||||||||
| 5274 | // icmp sle i1 A, B -> A | ~B | ||||||||
| 5275 | return BinaryOperator::CreateOr(Builder.CreateNot(B), A); | ||||||||
| 5276 | } | ||||||||
| 5277 | } | ||||||||
| 5278 | |||||||||
| 5279 | // Transform pattern like: | ||||||||
| 5280 | // (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X | ||||||||
| 5281 | // (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X | ||||||||
| 5282 | // Into: | ||||||||
| 5283 | // (X l>> Y) != 0 | ||||||||
| 5284 | // (X l>> Y) == 0 | ||||||||
| 5285 | static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp, | ||||||||
| 5286 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 5287 | ICmpInst::Predicate Pred, NewPred; | ||||||||
| 5288 | Value *X, *Y; | ||||||||
| 5289 | if (match(&Cmp, | ||||||||
| 5290 | m_c_ICmp(Pred, m_OneUse(m_Shl(m_One(), m_Value(Y))), m_Value(X)))) { | ||||||||
| 5291 | switch (Pred) { | ||||||||
| 5292 | case ICmpInst::ICMP_ULE: | ||||||||
| 5293 | NewPred = ICmpInst::ICMP_NE; | ||||||||
| 5294 | break; | ||||||||
| 5295 | case ICmpInst::ICMP_UGT: | ||||||||
| 5296 | NewPred = ICmpInst::ICMP_EQ; | ||||||||
| 5297 | break; | ||||||||
| 5298 | default: | ||||||||
| 5299 | return nullptr; | ||||||||
| 5300 | } | ||||||||
| 5301 | } else if (match(&Cmp, m_c_ICmp(Pred, | ||||||||
| 5302 | m_OneUse(m_CombineOr( | ||||||||
| 5303 | m_Not(m_Shl(m_AllOnes(), m_Value(Y))), | ||||||||
| 5304 | m_Add(m_Shl(m_One(), m_Value(Y)), | ||||||||
| 5305 | m_AllOnes()))), | ||||||||
| 5306 | m_Value(X)))) { | ||||||||
| 5307 | // The variant with 'add' is not canonical, (the variant with 'not' is) | ||||||||
| 5308 | // we only get it because it has extra uses, and can't be canonicalized, | ||||||||
| 5309 | |||||||||
| 5310 | switch (Pred) { | ||||||||
| 5311 | case ICmpInst::ICMP_ULT: | ||||||||
| 5312 | NewPred = ICmpInst::ICMP_NE; | ||||||||
| 5313 | break; | ||||||||
| 5314 | case ICmpInst::ICMP_UGE: | ||||||||
| 5315 | NewPred = ICmpInst::ICMP_EQ; | ||||||||
| 5316 | break; | ||||||||
| 5317 | default: | ||||||||
| 5318 | return nullptr; | ||||||||
| 5319 | } | ||||||||
| 5320 | } else | ||||||||
| 5321 | return nullptr; | ||||||||
| 5322 | |||||||||
| 5323 | Value *NewX = Builder.CreateLShr(X, Y, X->getName() + ".highbits"); | ||||||||
| 5324 | Constant *Zero = Constant::getNullValue(NewX->getType()); | ||||||||
| 5325 | return CmpInst::Create(Instruction::ICmp, NewPred, NewX, Zero); | ||||||||
| 5326 | } | ||||||||
| 5327 | |||||||||
| 5328 | static Instruction *foldVectorCmp(CmpInst &Cmp, | ||||||||
| 5329 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 5330 | const CmpInst::Predicate Pred = Cmp.getPredicate(); | ||||||||
| 5331 | Value *LHS = Cmp.getOperand(0), *RHS = Cmp.getOperand(1); | ||||||||
| 5332 | bool IsFP = isa<FCmpInst>(Cmp); | ||||||||
| 5333 | |||||||||
| 5334 | Value *V1, *V2; | ||||||||
| 5335 | Constant *M; | ||||||||
| 5336 | if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M)))) | ||||||||
| 5337 | return nullptr; | ||||||||
| 5338 | |||||||||
| 5339 | // If both arguments of the cmp are shuffles that use the same mask and | ||||||||
| 5340 | // shuffle within a single vector, move the shuffle after the cmp: | ||||||||
| 5341 | // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M | ||||||||
| 5342 | Type *V1Ty = V1->getType(); | ||||||||
| 5343 | if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) && | ||||||||
| 5344 | V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) { | ||||||||
| 5345 | Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, V2) | ||||||||
| 5346 | : Builder.CreateICmp(Pred, V1, V2); | ||||||||
| 5347 | return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M); | ||||||||
| 5348 | } | ||||||||
| 5349 | |||||||||
| 5350 | // Try to canonicalize compare with splatted operand and splat constant. | ||||||||
| 5351 | // TODO: We could generalize this for more than splats. See/use the code in | ||||||||
| 5352 | // InstCombiner::foldVectorBinop(). | ||||||||
| 5353 | Constant *C; | ||||||||
| 5354 | if (!LHS->hasOneUse() || !match(RHS, m_Constant(C))) | ||||||||
| 5355 | return nullptr; | ||||||||
| 5356 | |||||||||
| 5357 | // Length-changing splats are ok, so adjust the constants as needed: | ||||||||
| 5358 | // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M | ||||||||
| 5359 | Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true); | ||||||||
| 5360 | Constant *ScalarM = M->getSplatValue(/* AllowUndefs */ true); | ||||||||
| 5361 | if (ScalarC && ScalarM) { | ||||||||
| 5362 | // We allow undefs in matching, but this transform removes those for safety. | ||||||||
| 5363 | // Demanded elements analysis should be able to recover some/all of that. | ||||||||
| 5364 | C = ConstantVector::getSplat(V1Ty->getVectorNumElements(), ScalarC); | ||||||||
| 5365 | M = ConstantVector::getSplat(M->getType()->getVectorNumElements(), ScalarM); | ||||||||
| 5366 | Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C) | ||||||||
| 5367 | : Builder.CreateICmp(Pred, V1, C); | ||||||||
| 5368 | return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M); | ||||||||
| 5369 | } | ||||||||
| 5370 | |||||||||
| 5371 | return nullptr; | ||||||||
| 5372 | } | ||||||||
| 5373 | |||||||||
| 5374 | // extract(uadd.with.overflow(A, B), 0) ult A | ||||||||
| 5375 | // -> extract(uadd.with.overflow(A, B), 1) | ||||||||
| 5376 | static Instruction *foldICmpOfUAddOv(ICmpInst &I) { | ||||||||
| 5377 | CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 5378 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 5379 | |||||||||
| 5380 | Value *UAddOv; | ||||||||
| 5381 | Value *A, *B; | ||||||||
| 5382 | auto UAddOvResultPat = m_ExtractValue<0>( | ||||||||
| 5383 | m_Intrinsic<Intrinsic::uadd_with_overflow>(m_Value(A), m_Value(B))); | ||||||||
| 5384 | if (match(Op0, UAddOvResultPat) && | ||||||||
| 5385 | ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) || | ||||||||
| 5386 | (Pred == ICmpInst::ICMP_EQ && match(Op1, m_ZeroInt()) && | ||||||||
| 5387 | (match(A, m_One()) || match(B, m_One()))) || | ||||||||
| 5388 | (Pred == ICmpInst::ICMP_NE && match(Op1, m_AllOnes()) && | ||||||||
| 5389 | (match(A, m_AllOnes()) || match(B, m_AllOnes()))))) | ||||||||
| 5390 | // extract(uadd.with.overflow(A, B), 0) < A | ||||||||
| 5391 | // extract(uadd.with.overflow(A, 1), 0) == 0 | ||||||||
| 5392 | // extract(uadd.with.overflow(A, -1), 0) != -1 | ||||||||
| 5393 | UAddOv = cast<ExtractValueInst>(Op0)->getAggregateOperand(); | ||||||||
| 5394 | else if (match(Op1, UAddOvResultPat) && | ||||||||
| 5395 | Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B)) | ||||||||
| 5396 | // A > extract(uadd.with.overflow(A, B), 0) | ||||||||
| 5397 | UAddOv = cast<ExtractValueInst>(Op1)->getAggregateOperand(); | ||||||||
| 5398 | else | ||||||||
| 5399 | return nullptr; | ||||||||
| 5400 | |||||||||
| 5401 | return ExtractValueInst::Create(UAddOv, 1); | ||||||||
| 5402 | } | ||||||||
| 5403 | |||||||||
| 5404 | Instruction *InstCombiner::visitICmpInst(ICmpInst &I) { | ||||||||
| 5405 | bool Changed = false; | ||||||||
| 5406 | const SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||||||
| 5407 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 5408 | unsigned Op0Cplxity = getComplexity(Op0); | ||||||||
| 5409 | unsigned Op1Cplxity = getComplexity(Op1); | ||||||||
| 5410 | |||||||||
| 5411 | /// Orders the operands of the compare so that they are listed from most | ||||||||
| 5412 | /// complex to least complex. This puts constants before unary operators, | ||||||||
| 5413 | /// before binary operators. | ||||||||
| 5414 | if (Op0Cplxity < Op1Cplxity || | ||||||||
| 5415 | (Op0Cplxity == Op1Cplxity && swapMayExposeCSEOpportunities(Op0, Op1))) { | ||||||||
| 5416 | I.swapOperands(); | ||||||||
| 5417 | std::swap(Op0, Op1); | ||||||||
| 5418 | Changed = true; | ||||||||
| 5419 | } | ||||||||
| 5420 | |||||||||
| 5421 | if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, Q)) | ||||||||
| 5422 | return replaceInstUsesWith(I, V); | ||||||||
| 5423 | |||||||||
| 5424 | // Comparing -val or val with non-zero is the same as just comparing val | ||||||||
| 5425 | // ie, abs(val) != 0 -> val != 0 | ||||||||
| 5426 | if (I.getPredicate() == ICmpInst::ICMP_NE && match(Op1, m_Zero())) { | ||||||||
| 5427 | Value *Cond, *SelectTrue, *SelectFalse; | ||||||||
| 5428 | if (match(Op0, m_Select(m_Value(Cond), m_Value(SelectTrue), | ||||||||
| 5429 | m_Value(SelectFalse)))) { | ||||||||
| 5430 | if (Value *V = dyn_castNegVal(SelectTrue)) { | ||||||||
| 5431 | if (V == SelectFalse) | ||||||||
| 5432 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | ||||||||
| 5433 | } | ||||||||
| 5434 | else if (Value *V = dyn_castNegVal(SelectFalse)) { | ||||||||
| 5435 | if (V == SelectTrue) | ||||||||
| 5436 | return CmpInst::Create(Instruction::ICmp, I.getPredicate(), V, Op1); | ||||||||
| 5437 | } | ||||||||
| 5438 | } | ||||||||
| 5439 | } | ||||||||
| 5440 | |||||||||
| 5441 | if (Op0->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 5442 | if (Instruction *Res = canonicalizeICmpBool(I, Builder)) | ||||||||
| 5443 | return Res; | ||||||||
| 5444 | |||||||||
| 5445 | if (ICmpInst *NewICmp = canonicalizeCmpWithConstant(I)) | ||||||||
| 5446 | return NewICmp; | ||||||||
| 5447 | |||||||||
| 5448 | if (Instruction *Res = foldICmpWithConstant(I)) | ||||||||
| 5449 | return Res; | ||||||||
| 5450 | |||||||||
| 5451 | if (Instruction *Res = foldICmpWithDominatingICmp(I)) | ||||||||
| 5452 | return Res; | ||||||||
| 5453 | |||||||||
| 5454 | if (Instruction *Res = foldICmpBinOp(I, Q)) | ||||||||
| 5455 | return Res; | ||||||||
| 5456 | |||||||||
| 5457 | if (Instruction *Res = foldICmpUsingKnownBits(I)) | ||||||||
| 5458 | return Res; | ||||||||
| 5459 | |||||||||
| 5460 | // Test if the ICmpInst instruction is used exclusively by a select as | ||||||||
| 5461 | // part of a minimum or maximum operation. If so, refrain from doing | ||||||||
| 5462 | // any other folding. This helps out other analyses which understand | ||||||||
| 5463 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | ||||||||
| 5464 | // and CodeGen. And in this case, at least one of the comparison | ||||||||
| 5465 | // operands has at least one user besides the compare (the select), | ||||||||
| 5466 | // which would often largely negate the benefit of folding anyway. | ||||||||
| 5467 | // | ||||||||
| 5468 | // Do the same for the other patterns recognized by matchSelectPattern. | ||||||||
| 5469 | if (I.hasOneUse()) | ||||||||
| 5470 | if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { | ||||||||
| 5471 | Value *A, *B; | ||||||||
| 5472 | SelectPatternResult SPR = matchSelectPattern(SI, A, B); | ||||||||
| 5473 | if (SPR.Flavor != SPF_UNKNOWN) | ||||||||
| 5474 | return nullptr; | ||||||||
| 5475 | } | ||||||||
| 5476 | |||||||||
| 5477 | // Do this after checking for min/max to prevent infinite looping. | ||||||||
| 5478 | if (Instruction *Res = foldICmpWithZero(I)) | ||||||||
| 5479 | return Res; | ||||||||
| 5480 | |||||||||
| 5481 | // FIXME: We only do this after checking for min/max to prevent infinite | ||||||||
| 5482 | // looping caused by a reverse canonicalization of these patterns for min/max. | ||||||||
| 5483 | // FIXME: The organization of folds is a mess. These would naturally go into | ||||||||
| 5484 | // canonicalizeCmpWithConstant(), but we can't move all of the above folds | ||||||||
| 5485 | // down here after the min/max restriction. | ||||||||
| 5486 | ICmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 5487 | const APInt *C; | ||||||||
| 5488 | if (match(Op1, m_APInt(C))) { | ||||||||
| 5489 | // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set | ||||||||
| 5490 | if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) { | ||||||||
| 5491 | Constant *Zero = Constant::getNullValue(Op0->getType()); | ||||||||
| 5492 | return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero); | ||||||||
| 5493 | } | ||||||||
| 5494 | |||||||||
| 5495 | // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear | ||||||||
| 5496 | if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) { | ||||||||
| 5497 | Constant *AllOnes = Constant::getAllOnesValue(Op0->getType()); | ||||||||
| 5498 | return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes); | ||||||||
| 5499 | } | ||||||||
| 5500 | } | ||||||||
| 5501 | |||||||||
| 5502 | if (Instruction *Res = foldICmpInstWithConstant(I)) | ||||||||
| 5503 | return Res; | ||||||||
| 5504 | |||||||||
| 5505 | // Try to match comparison as a sign bit test. Intentionally do this after | ||||||||
| 5506 | // foldICmpInstWithConstant() to potentially let other folds to happen first. | ||||||||
| 5507 | if (Instruction *New = foldSignBitTest(I)) | ||||||||
| 5508 | return New; | ||||||||
| 5509 | |||||||||
| 5510 | if (Instruction *Res = foldICmpInstWithConstantNotInt(I)) | ||||||||
| 5511 | return Res; | ||||||||
| 5512 | |||||||||
| 5513 | // If we can optimize a 'icmp GEP, P' or 'icmp P, GEP', do so now. | ||||||||
| 5514 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op0)) | ||||||||
| 5515 | if (Instruction *NI = foldGEPICmp(GEP, Op1, I.getPredicate(), I)) | ||||||||
| 5516 | return NI; | ||||||||
| 5517 | if (GEPOperator *GEP = dyn_cast<GEPOperator>(Op1)) | ||||||||
| 5518 | if (Instruction *NI = foldGEPICmp(GEP, Op0, | ||||||||
| 5519 | ICmpInst::getSwappedPredicate(I.getPredicate()), I)) | ||||||||
| 5520 | return NI; | ||||||||
| 5521 | |||||||||
| 5522 | // Try to optimize equality comparisons against alloca-based pointers. | ||||||||
| 5523 | if (Op0->getType()->isPointerTy() && I.isEquality()) { | ||||||||
| 5524 | assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?")((Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("Op1->getType()->isPointerTy() && \"Comparing pointer with non-pointer?\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5524, __PRETTY_FUNCTION__)); | ||||||||
| 5525 | if (auto *Alloca = dyn_cast<AllocaInst>(GetUnderlyingObject(Op0, DL))) | ||||||||
| 5526 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op1)) | ||||||||
| 5527 | return New; | ||||||||
| 5528 | if (auto *Alloca = dyn_cast<AllocaInst>(GetUnderlyingObject(Op1, DL))) | ||||||||
| 5529 | if (Instruction *New = foldAllocaCmp(I, Alloca, Op0)) | ||||||||
| 5530 | return New; | ||||||||
| 5531 | } | ||||||||
| 5532 | |||||||||
| 5533 | if (Instruction *Res = foldICmpBitCast(I, Builder)) | ||||||||
| 5534 | return Res; | ||||||||
| 5535 | |||||||||
| 5536 | if (Instruction *R = foldICmpWithCastOp(I)) | ||||||||
| 5537 | return R; | ||||||||
| 5538 | |||||||||
| 5539 | if (Instruction *Res = foldICmpWithMinMax(I)) | ||||||||
| 5540 | return Res; | ||||||||
| 5541 | |||||||||
| 5542 | { | ||||||||
| 5543 | Value *A, *B; | ||||||||
| 5544 | // Transform (A & ~B) == 0 --> (A & B) != 0 | ||||||||
| 5545 | // and (A & ~B) != 0 --> (A & B) == 0 | ||||||||
| 5546 | // if A is a power of 2. | ||||||||
| 5547 | if (match(Op0, m_And(m_Value(A), m_Not(m_Value(B)))) && | ||||||||
| 5548 | match(Op1, m_Zero()) && | ||||||||
| 5549 | isKnownToBeAPowerOfTwo(A, false, 0, &I) && I.isEquality()) | ||||||||
| 5550 | return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(A, B), | ||||||||
| 5551 | Op1); | ||||||||
| 5552 | |||||||||
| 5553 | // ~X < ~Y --> Y < X | ||||||||
| 5554 | // ~X < C --> X > ~C | ||||||||
| 5555 | if (match(Op0, m_Not(m_Value(A)))) { | ||||||||
| 5556 | if (match(Op1, m_Not(m_Value(B)))) | ||||||||
| 5557 | return new ICmpInst(I.getPredicate(), B, A); | ||||||||
| 5558 | |||||||||
| 5559 | const APInt *C; | ||||||||
| 5560 | if (match(Op1, m_APInt(C))) | ||||||||
| 5561 | return new ICmpInst(I.getSwappedPredicate(), A, | ||||||||
| 5562 | ConstantInt::get(Op1->getType(), ~(*C))); | ||||||||
| 5563 | } | ||||||||
| 5564 | |||||||||
| 5565 | Instruction *AddI = nullptr; | ||||||||
| 5566 | if (match(&I, m_UAddWithOverflow(m_Value(A), m_Value(B), | ||||||||
| 5567 | m_Instruction(AddI))) && | ||||||||
| 5568 | isa<IntegerType>(A->getType())) { | ||||||||
| 5569 | Value *Result; | ||||||||
| 5570 | Constant *Overflow; | ||||||||
| 5571 | // m_UAddWithOverflow can match patterns that do not include an explicit | ||||||||
| 5572 | // "add" instruction, so check the opcode of the matched op. | ||||||||
| 5573 | if (AddI->getOpcode() == Instruction::Add && | ||||||||
| 5574 | OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, A, B, *AddI, | ||||||||
| 5575 | Result, Overflow)) { | ||||||||
| 5576 | replaceInstUsesWith(*AddI, Result); | ||||||||
| 5577 | return replaceInstUsesWith(I, Overflow); | ||||||||
| 5578 | } | ||||||||
| 5579 | } | ||||||||
| 5580 | |||||||||
| 5581 | // (zext a) * (zext b) --> llvm.umul.with.overflow. | ||||||||
| 5582 | if (match(Op0, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | ||||||||
| 5583 | if (Instruction *R = processUMulZExtIdiom(I, Op0, Op1, *this)) | ||||||||
| 5584 | return R; | ||||||||
| 5585 | } | ||||||||
| 5586 | if (match(Op1, m_Mul(m_ZExt(m_Value(A)), m_ZExt(m_Value(B))))) { | ||||||||
| 5587 | if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this)) | ||||||||
| 5588 | return R; | ||||||||
| 5589 | } | ||||||||
| 5590 | } | ||||||||
| 5591 | |||||||||
| 5592 | if (Instruction *Res = foldICmpEquality(I)) | ||||||||
| 5593 | return Res; | ||||||||
| 5594 | |||||||||
| 5595 | if (Instruction *Res = foldICmpOfUAddOv(I)) | ||||||||
| 5596 | return Res; | ||||||||
| 5597 | |||||||||
| 5598 | // The 'cmpxchg' instruction returns an aggregate containing the old value and | ||||||||
| 5599 | // an i1 which indicates whether or not we successfully did the swap. | ||||||||
| 5600 | // | ||||||||
| 5601 | // Replace comparisons between the old value and the expected value with the | ||||||||
| 5602 | // indicator that 'cmpxchg' returns. | ||||||||
| 5603 | // | ||||||||
| 5604 | // N.B. This transform is only valid when the 'cmpxchg' is not permitted to | ||||||||
| 5605 | // spuriously fail. In those cases, the old value may equal the expected | ||||||||
| 5606 | // value but it is possible for the swap to not occur. | ||||||||
| 5607 | if (I.getPredicate() == ICmpInst::ICMP_EQ) | ||||||||
| 5608 | if (auto *EVI = dyn_cast<ExtractValueInst>(Op0)) | ||||||||
| 5609 | if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(EVI->getAggregateOperand())) | ||||||||
| 5610 | if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 && | ||||||||
| 5611 | !ACXI->isWeak()) | ||||||||
| 5612 | return ExtractValueInst::Create(ACXI, 1); | ||||||||
| 5613 | |||||||||
| 5614 | { | ||||||||
| 5615 | Value *X; | ||||||||
| 5616 | const APInt *C; | ||||||||
| 5617 | // icmp X+Cst, X | ||||||||
| 5618 | if (match(Op0, m_Add(m_Value(X), m_APInt(C))) && Op1 == X) | ||||||||
| 5619 | return foldICmpAddOpConst(X, *C, I.getPredicate()); | ||||||||
| 5620 | |||||||||
| 5621 | // icmp X, X+Cst | ||||||||
| 5622 | if (match(Op1, m_Add(m_Value(X), m_APInt(C))) && Op0 == X) | ||||||||
| 5623 | return foldICmpAddOpConst(X, *C, I.getSwappedPredicate()); | ||||||||
| 5624 | } | ||||||||
| 5625 | |||||||||
| 5626 | if (Instruction *Res = foldICmpWithHighBitMask(I, Builder)) | ||||||||
| 5627 | return Res; | ||||||||
| 5628 | |||||||||
| 5629 | if (I.getType()->isVectorTy()) | ||||||||
| 5630 | if (Instruction *Res = foldVectorCmp(I, Builder)) | ||||||||
| 5631 | return Res; | ||||||||
| 5632 | |||||||||
| 5633 | return Changed ? &I : nullptr; | ||||||||
| 5634 | } | ||||||||
| 5635 | |||||||||
| 5636 | /// Fold fcmp ([us]itofp x, cst) if possible. | ||||||||
| 5637 | Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, | ||||||||
| 5638 | Constant *RHSC) { | ||||||||
| 5639 | if (!isa<ConstantFP>(RHSC)) return nullptr; | ||||||||
| 5640 | const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF(); | ||||||||
| 5641 | |||||||||
| 5642 | // Get the width of the mantissa. We don't want to hack on conversions that | ||||||||
| 5643 | // might lose information from the integer, e.g. "i64 -> float" | ||||||||
| 5644 | int MantissaWidth = LHSI->getType()->getFPMantissaWidth(); | ||||||||
| 5645 | if (MantissaWidth == -1) return nullptr; // Unknown. | ||||||||
| 5646 | |||||||||
| 5647 | IntegerType *IntTy = cast<IntegerType>(LHSI->getOperand(0)->getType()); | ||||||||
| 5648 | |||||||||
| 5649 | bool LHSUnsigned = isa<UIToFPInst>(LHSI); | ||||||||
| 5650 | |||||||||
| 5651 | if (I.isEquality()) { | ||||||||
| 5652 | FCmpInst::Predicate P = I.getPredicate(); | ||||||||
| 5653 | bool IsExact = false; | ||||||||
| 5654 | APSInt RHSCvt(IntTy->getBitWidth(), LHSUnsigned); | ||||||||
| 5655 | RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); | ||||||||
| 5656 | |||||||||
| 5657 | // If the floating point constant isn't an integer value, we know if we will | ||||||||
| 5658 | // ever compare equal / not equal to it. | ||||||||
| 5659 | if (!IsExact) { | ||||||||
| 5660 | // TODO: Can never be -0.0 and other non-representable values | ||||||||
| 5661 | APFloat RHSRoundInt(RHS); | ||||||||
| 5662 | RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); | ||||||||
| 5663 | if (RHS.compare(RHSRoundInt) != APFloat::cmpEqual) { | ||||||||
| 5664 | if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) | ||||||||
| 5665 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5666 | |||||||||
| 5667 | assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE)((P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE) ? static_cast <void> (0) : __assert_fail ("P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5667, __PRETTY_FUNCTION__)); | ||||||||
| 5668 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5669 | } | ||||||||
| 5670 | } | ||||||||
| 5671 | |||||||||
| 5672 | // TODO: If the constant is exactly representable, is it always OK to do | ||||||||
| 5673 | // equality compares as integer? | ||||||||
| 5674 | } | ||||||||
| 5675 | |||||||||
| 5676 | // Check to see that the input is converted from an integer type that is small | ||||||||
| 5677 | // enough that preserves all bits. TODO: check here for "known" sign bits. | ||||||||
| 5678 | // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e. | ||||||||
| 5679 | unsigned InputSize = IntTy->getScalarSizeInBits(); | ||||||||
| 5680 | |||||||||
| 5681 | // Following test does NOT adjust InputSize downwards for signed inputs, | ||||||||
| 5682 | // because the most negative value still requires all the mantissa bits | ||||||||
| 5683 | // to distinguish it from one less than that value. | ||||||||
| 5684 | if ((int)InputSize > MantissaWidth) { | ||||||||
| 5685 | // Conversion would lose accuracy. Check if loss can impact comparison. | ||||||||
| 5686 | int Exp = ilogb(RHS); | ||||||||
| 5687 | if (Exp == APFloat::IEK_Inf) { | ||||||||
| 5688 | int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics())); | ||||||||
| 5689 | if (MaxExponent < (int)InputSize - !LHSUnsigned) | ||||||||
| 5690 | // Conversion could create infinity. | ||||||||
| 5691 | return nullptr; | ||||||||
| 5692 | } else { | ||||||||
| 5693 | // Note that if RHS is zero or NaN, then Exp is negative | ||||||||
| 5694 | // and first condition is trivially false. | ||||||||
| 5695 | if (MantissaWidth <= Exp && Exp <= (int)InputSize - !LHSUnsigned) | ||||||||
| 5696 | // Conversion could affect comparison. | ||||||||
| 5697 | return nullptr; | ||||||||
| 5698 | } | ||||||||
| 5699 | } | ||||||||
| 5700 | |||||||||
| 5701 | // Otherwise, we can potentially simplify the comparison. We know that it | ||||||||
| 5702 | // will always come through as an integer value and we know the constant is | ||||||||
| 5703 | // not a NAN (it would have been previously simplified). | ||||||||
| 5704 | assert(!RHS.isNaN() && "NaN comparison not already folded!")((!RHS.isNaN() && "NaN comparison not already folded!" ) ? static_cast<void> (0) : __assert_fail ("!RHS.isNaN() && \"NaN comparison not already folded!\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5704, __PRETTY_FUNCTION__)); | ||||||||
| 5705 | |||||||||
| 5706 | ICmpInst::Predicate Pred; | ||||||||
| 5707 | switch (I.getPredicate()) { | ||||||||
| 5708 | default: llvm_unreachable("Unexpected predicate!")::llvm::llvm_unreachable_internal("Unexpected predicate!", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5708); | ||||||||
| 5709 | case FCmpInst::FCMP_UEQ: | ||||||||
| 5710 | case FCmpInst::FCMP_OEQ: | ||||||||
| 5711 | Pred = ICmpInst::ICMP_EQ; | ||||||||
| 5712 | break; | ||||||||
| 5713 | case FCmpInst::FCMP_UGT: | ||||||||
| 5714 | case FCmpInst::FCMP_OGT: | ||||||||
| 5715 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT; | ||||||||
| 5716 | break; | ||||||||
| 5717 | case FCmpInst::FCMP_UGE: | ||||||||
| 5718 | case FCmpInst::FCMP_OGE: | ||||||||
| 5719 | Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE; | ||||||||
| 5720 | break; | ||||||||
| 5721 | case FCmpInst::FCMP_ULT: | ||||||||
| 5722 | case FCmpInst::FCMP_OLT: | ||||||||
| 5723 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT; | ||||||||
| 5724 | break; | ||||||||
| 5725 | case FCmpInst::FCMP_ULE: | ||||||||
| 5726 | case FCmpInst::FCMP_OLE: | ||||||||
| 5727 | Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE; | ||||||||
| 5728 | break; | ||||||||
| 5729 | case FCmpInst::FCMP_UNE: | ||||||||
| 5730 | case FCmpInst::FCMP_ONE: | ||||||||
| 5731 | Pred = ICmpInst::ICMP_NE; | ||||||||
| 5732 | break; | ||||||||
| 5733 | case FCmpInst::FCMP_ORD: | ||||||||
| 5734 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5735 | case FCmpInst::FCMP_UNO: | ||||||||
| 5736 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5737 | } | ||||||||
| 5738 | |||||||||
| 5739 | // Now we know that the APFloat is a normal number, zero or inf. | ||||||||
| 5740 | |||||||||
| 5741 | // See if the FP constant is too large for the integer. For example, | ||||||||
| 5742 | // comparing an i8 to 300.0. | ||||||||
| 5743 | unsigned IntWidth = IntTy->getScalarSizeInBits(); | ||||||||
| 5744 | |||||||||
| 5745 | if (!LHSUnsigned) { | ||||||||
| 5746 | // If the RHS value is > SignedMax, fold the comparison. This handles +INF | ||||||||
| 5747 | // and large values. | ||||||||
| 5748 | APFloat SMax(RHS.getSemantics()); | ||||||||
| 5749 | SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true, | ||||||||
| 5750 | APFloat::rmNearestTiesToEven); | ||||||||
| 5751 | if (SMax.compare(RHS) == APFloat::cmpLessThan) { // smax < 13123.0 | ||||||||
| 5752 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || | ||||||||
| 5753 | Pred == ICmpInst::ICMP_SLE) | ||||||||
| 5754 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5755 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5756 | } | ||||||||
| 5757 | } else { | ||||||||
| 5758 | // If the RHS value is > UnsignedMax, fold the comparison. This handles | ||||||||
| 5759 | // +INF and large values. | ||||||||
| 5760 | APFloat UMax(RHS.getSemantics()); | ||||||||
| 5761 | UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false, | ||||||||
| 5762 | APFloat::rmNearestTiesToEven); | ||||||||
| 5763 | if (UMax.compare(RHS) == APFloat::cmpLessThan) { // umax < 13123.0 | ||||||||
| 5764 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || | ||||||||
| 5765 | Pred == ICmpInst::ICMP_ULE) | ||||||||
| 5766 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5767 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5768 | } | ||||||||
| 5769 | } | ||||||||
| 5770 | |||||||||
| 5771 | if (!LHSUnsigned) { | ||||||||
| 5772 | // See if the RHS value is < SignedMin. | ||||||||
| 5773 | APFloat SMin(RHS.getSemantics()); | ||||||||
| 5774 | SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true, | ||||||||
| 5775 | APFloat::rmNearestTiesToEven); | ||||||||
| 5776 | if (SMin.compare(RHS) == APFloat::cmpGreaterThan) { // smin > 12312.0 | ||||||||
| 5777 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || | ||||||||
| 5778 | Pred == ICmpInst::ICMP_SGE) | ||||||||
| 5779 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5780 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5781 | } | ||||||||
| 5782 | } else { | ||||||||
| 5783 | // See if the RHS value is < UnsignedMin. | ||||||||
| 5784 | APFloat SMin(RHS.getSemantics()); | ||||||||
| 5785 | SMin.convertFromAPInt(APInt::getMinValue(IntWidth), true, | ||||||||
| 5786 | APFloat::rmNearestTiesToEven); | ||||||||
| 5787 | if (SMin.compare(RHS) == APFloat::cmpGreaterThan) { // umin > 12312.0 | ||||||||
| 5788 | if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || | ||||||||
| 5789 | Pred == ICmpInst::ICMP_UGE) | ||||||||
| 5790 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5791 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5792 | } | ||||||||
| 5793 | } | ||||||||
| 5794 | |||||||||
| 5795 | // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or | ||||||||
| 5796 | // [0, UMAX], but it may still be fractional. See if it is fractional by | ||||||||
| 5797 | // casting the FP value to the integer value and back, checking for equality. | ||||||||
| 5798 | // Don't do this for zero, because -0.0 is not fractional. | ||||||||
| 5799 | Constant *RHSInt = LHSUnsigned | ||||||||
| 5800 | ? ConstantExpr::getFPToUI(RHSC, IntTy) | ||||||||
| 5801 | : ConstantExpr::getFPToSI(RHSC, IntTy); | ||||||||
| 5802 | if (!RHS.isZero()) { | ||||||||
| 5803 | bool Equal = LHSUnsigned | ||||||||
| 5804 | ? ConstantExpr::getUIToFP(RHSInt, RHSC->getType()) == RHSC | ||||||||
| 5805 | : ConstantExpr::getSIToFP(RHSInt, RHSC->getType()) == RHSC; | ||||||||
| 5806 | if (!Equal) { | ||||||||
| 5807 | // If we had a comparison against a fractional value, we have to adjust | ||||||||
| 5808 | // the compare predicate and sometimes the value. RHSC is rounded towards | ||||||||
| 5809 | // zero at this point. | ||||||||
| 5810 | switch (Pred) { | ||||||||
| 5811 | default: llvm_unreachable("Unexpected integer comparison!")::llvm::llvm_unreachable_internal("Unexpected integer comparison!" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5811); | ||||||||
| 5812 | case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true | ||||||||
| 5813 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5814 | case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false | ||||||||
| 5815 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5816 | case ICmpInst::ICMP_ULE: | ||||||||
| 5817 | // (float)int <= 4.4 --> int <= 4 | ||||||||
| 5818 | // (float)int <= -4.4 --> false | ||||||||
| 5819 | if (RHS.isNegative()) | ||||||||
| 5820 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5821 | break; | ||||||||
| 5822 | case ICmpInst::ICMP_SLE: | ||||||||
| 5823 | // (float)int <= 4.4 --> int <= 4 | ||||||||
| 5824 | // (float)int <= -4.4 --> int < -4 | ||||||||
| 5825 | if (RHS.isNegative()) | ||||||||
| 5826 | Pred = ICmpInst::ICMP_SLT; | ||||||||
| 5827 | break; | ||||||||
| 5828 | case ICmpInst::ICMP_ULT: | ||||||||
| 5829 | // (float)int < -4.4 --> false | ||||||||
| 5830 | // (float)int < 4.4 --> int <= 4 | ||||||||
| 5831 | if (RHS.isNegative()) | ||||||||
| 5832 | return replaceInstUsesWith(I, Builder.getFalse()); | ||||||||
| 5833 | Pred = ICmpInst::ICMP_ULE; | ||||||||
| 5834 | break; | ||||||||
| 5835 | case ICmpInst::ICMP_SLT: | ||||||||
| 5836 | // (float)int < -4.4 --> int < -4 | ||||||||
| 5837 | // (float)int < 4.4 --> int <= 4 | ||||||||
| 5838 | if (!RHS.isNegative()) | ||||||||
| 5839 | Pred = ICmpInst::ICMP_SLE; | ||||||||
| 5840 | break; | ||||||||
| 5841 | case ICmpInst::ICMP_UGT: | ||||||||
| 5842 | // (float)int > 4.4 --> int > 4 | ||||||||
| 5843 | // (float)int > -4.4 --> true | ||||||||
| 5844 | if (RHS.isNegative()) | ||||||||
| 5845 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5846 | break; | ||||||||
| 5847 | case ICmpInst::ICMP_SGT: | ||||||||
| 5848 | // (float)int > 4.4 --> int > 4 | ||||||||
| 5849 | // (float)int > -4.4 --> int >= -4 | ||||||||
| 5850 | if (RHS.isNegative()) | ||||||||
| 5851 | Pred = ICmpInst::ICMP_SGE; | ||||||||
| 5852 | break; | ||||||||
| 5853 | case ICmpInst::ICMP_UGE: | ||||||||
| 5854 | // (float)int >= -4.4 --> true | ||||||||
| 5855 | // (float)int >= 4.4 --> int > 4 | ||||||||
| 5856 | if (RHS.isNegative()) | ||||||||
| 5857 | return replaceInstUsesWith(I, Builder.getTrue()); | ||||||||
| 5858 | Pred = ICmpInst::ICMP_UGT; | ||||||||
| 5859 | break; | ||||||||
| 5860 | case ICmpInst::ICMP_SGE: | ||||||||
| 5861 | // (float)int >= -4.4 --> int >= -4 | ||||||||
| 5862 | // (float)int >= 4.4 --> int > 4 | ||||||||
| 5863 | if (!RHS.isNegative()) | ||||||||
| 5864 | Pred = ICmpInst::ICMP_SGT; | ||||||||
| 5865 | break; | ||||||||
| 5866 | } | ||||||||
| 5867 | } | ||||||||
| 5868 | } | ||||||||
| 5869 | |||||||||
| 5870 | // Lower this FP comparison into an appropriate integer version of the | ||||||||
| 5871 | // comparison. | ||||||||
| 5872 | return new ICmpInst(Pred, LHSI->getOperand(0), RHSInt); | ||||||||
| 5873 | } | ||||||||
| 5874 | |||||||||
| 5875 | /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary. | ||||||||
| 5876 | static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI, | ||||||||
| 5877 | Constant *RHSC) { | ||||||||
| 5878 | // When C is not 0.0 and infinities are not allowed: | ||||||||
| 5879 | // (C / X) < 0.0 is a sign-bit test of X | ||||||||
| 5880 | // (C / X) < 0.0 --> X < 0.0 (if C is positive) | ||||||||
| 5881 | // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate) | ||||||||
| 5882 | // | ||||||||
| 5883 | // Proof: | ||||||||
| 5884 | // Multiply (C / X) < 0.0 by X * X / C. | ||||||||
| 5885 | // - X is non zero, if it is the flag 'ninf' is violated. | ||||||||
| 5886 | // - C defines the sign of X * X * C. Thus it also defines whether to swap | ||||||||
| 5887 | // the predicate. C is also non zero by definition. | ||||||||
| 5888 | // | ||||||||
| 5889 | // Thus X * X / C is non zero and the transformation is valid. [qed] | ||||||||
| 5890 | |||||||||
| 5891 | FCmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 5892 | |||||||||
| 5893 | // Check that predicates are valid. | ||||||||
| 5894 | if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) && | ||||||||
| 5895 | (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE)) | ||||||||
| 5896 | return nullptr; | ||||||||
| 5897 | |||||||||
| 5898 | // Check that RHS operand is zero. | ||||||||
| 5899 | if (!match(RHSC, m_AnyZeroFP())) | ||||||||
| 5900 | return nullptr; | ||||||||
| 5901 | |||||||||
| 5902 | // Check fastmath flags ('ninf'). | ||||||||
| 5903 | if (!LHSI->hasNoInfs() || !I.hasNoInfs()) | ||||||||
| 5904 | return nullptr; | ||||||||
| 5905 | |||||||||
| 5906 | // Check the properties of the dividend. It must not be zero to avoid a | ||||||||
| 5907 | // division by zero (see Proof). | ||||||||
| 5908 | const APFloat *C; | ||||||||
| 5909 | if (!match(LHSI->getOperand(0), m_APFloat(C))) | ||||||||
| 5910 | return nullptr; | ||||||||
| 5911 | |||||||||
| 5912 | if (C->isZero()) | ||||||||
| 5913 | return nullptr; | ||||||||
| 5914 | |||||||||
| 5915 | // Get swapped predicate if necessary. | ||||||||
| 5916 | if (C->isNegative()) | ||||||||
| 5917 | Pred = I.getSwappedPredicate(); | ||||||||
| 5918 | |||||||||
| 5919 | return new FCmpInst(Pred, LHSI->getOperand(1), RHSC, "", &I); | ||||||||
| 5920 | } | ||||||||
| 5921 | |||||||||
| 5922 | /// Optimize fabs(X) compared with zero. | ||||||||
| 5923 | static Instruction *foldFabsWithFcmpZero(FCmpInst &I) { | ||||||||
| 5924 | Value *X; | ||||||||
| 5925 | if (!match(I.getOperand(0), m_Intrinsic<Intrinsic::fabs>(m_Value(X))) || | ||||||||
| 5926 | !match(I.getOperand(1), m_PosZeroFP())) | ||||||||
| 5927 | return nullptr; | ||||||||
| 5928 | |||||||||
| 5929 | auto replacePredAndOp0 = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) { | ||||||||
| 5930 | I->setPredicate(P); | ||||||||
| 5931 | I->setOperand(0, X); | ||||||||
| 5932 | return I; | ||||||||
| 5933 | }; | ||||||||
| 5934 | |||||||||
| 5935 | switch (I.getPredicate()) { | ||||||||
| 5936 | case FCmpInst::FCMP_UGE: | ||||||||
| 5937 | case FCmpInst::FCMP_OLT: | ||||||||
| 5938 | // fabs(X) >= 0.0 --> true | ||||||||
| 5939 | // fabs(X) < 0.0 --> false | ||||||||
| 5940 | llvm_unreachable("fcmp should have simplified")::llvm::llvm_unreachable_internal("fcmp should have simplified" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5940); | ||||||||
| 5941 | |||||||||
| 5942 | case FCmpInst::FCMP_OGT: | ||||||||
| 5943 | // fabs(X) > 0.0 --> X != 0.0 | ||||||||
| 5944 | return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X); | ||||||||
| 5945 | |||||||||
| 5946 | case FCmpInst::FCMP_UGT: | ||||||||
| 5947 | // fabs(X) u> 0.0 --> X u!= 0.0 | ||||||||
| 5948 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X); | ||||||||
| 5949 | |||||||||
| 5950 | case FCmpInst::FCMP_OLE: | ||||||||
| 5951 | // fabs(X) <= 0.0 --> X == 0.0 | ||||||||
| 5952 | return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X); | ||||||||
| 5953 | |||||||||
| 5954 | case FCmpInst::FCMP_ULE: | ||||||||
| 5955 | // fabs(X) u<= 0.0 --> X u== 0.0 | ||||||||
| 5956 | return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X); | ||||||||
| 5957 | |||||||||
| 5958 | case FCmpInst::FCMP_OGE: | ||||||||
| 5959 | // fabs(X) >= 0.0 --> !isnan(X) | ||||||||
| 5960 | assert(!I.hasNoNaNs() && "fcmp should have simplified")((!I.hasNoNaNs() && "fcmp should have simplified") ? static_cast <void> (0) : __assert_fail ("!I.hasNoNaNs() && \"fcmp should have simplified\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5960, __PRETTY_FUNCTION__)); | ||||||||
| 5961 | return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X); | ||||||||
| 5962 | |||||||||
| 5963 | case FCmpInst::FCMP_ULT: | ||||||||
| 5964 | // fabs(X) u< 0.0 --> isnan(X) | ||||||||
| 5965 | assert(!I.hasNoNaNs() && "fcmp should have simplified")((!I.hasNoNaNs() && "fcmp should have simplified") ? static_cast <void> (0) : __assert_fail ("!I.hasNoNaNs() && \"fcmp should have simplified\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 5965, __PRETTY_FUNCTION__)); | ||||||||
| 5966 | return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X); | ||||||||
| 5967 | |||||||||
| 5968 | case FCmpInst::FCMP_OEQ: | ||||||||
| 5969 | case FCmpInst::FCMP_UEQ: | ||||||||
| 5970 | case FCmpInst::FCMP_ONE: | ||||||||
| 5971 | case FCmpInst::FCMP_UNE: | ||||||||
| 5972 | case FCmpInst::FCMP_ORD: | ||||||||
| 5973 | case FCmpInst::FCMP_UNO: | ||||||||
| 5974 | // Look through the fabs() because it doesn't change anything but the sign. | ||||||||
| 5975 | // fabs(X) == 0.0 --> X == 0.0, | ||||||||
| 5976 | // fabs(X) != 0.0 --> X != 0.0 | ||||||||
| 5977 | // isnan(fabs(X)) --> isnan(X) | ||||||||
| 5978 | // !isnan(fabs(X) --> !isnan(X) | ||||||||
| 5979 | return replacePredAndOp0(&I, I.getPredicate(), X); | ||||||||
| 5980 | |||||||||
| 5981 | default: | ||||||||
| 5982 | return nullptr; | ||||||||
| 5983 | } | ||||||||
| 5984 | } | ||||||||
| 5985 | |||||||||
| 5986 | Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) { | ||||||||
| 5987 | bool Changed = false; | ||||||||
| 5988 | |||||||||
| 5989 | /// Orders the operands of the compare so that they are listed from most | ||||||||
| 5990 | /// complex to least complex. This puts constants before unary operators, | ||||||||
| 5991 | /// before binary operators. | ||||||||
| 5992 | if (getComplexity(I.getOperand(0)) < getComplexity(I.getOperand(1))) { | ||||||||
| 5993 | I.swapOperands(); | ||||||||
| 5994 | Changed = true; | ||||||||
| 5995 | } | ||||||||
| 5996 | |||||||||
| 5997 | const CmpInst::Predicate Pred = I.getPredicate(); | ||||||||
| 5998 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 5999 | if (Value *V = SimplifyFCmpInst(Pred, Op0, Op1, I.getFastMathFlags(), | ||||||||
| 6000 | SQ.getWithInstruction(&I))) | ||||||||
| 6001 | return replaceInstUsesWith(I, V); | ||||||||
| 6002 | |||||||||
| 6003 | // Simplify 'fcmp pred X, X' | ||||||||
| 6004 | Type *OpType = Op0->getType(); | ||||||||
| 6005 | assert(OpType == Op1->getType() && "fcmp with different-typed operands?")((OpType == Op1->getType() && "fcmp with different-typed operands?" ) ? static_cast<void> (0) : __assert_fail ("OpType == Op1->getType() && \"fcmp with different-typed operands?\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp" , 6005, __PRETTY_FUNCTION__)); | ||||||||
| 6006 | if (Op0 == Op1) { | ||||||||
| 6007 | switch (Pred) { | ||||||||
| 6008 | default: break; | ||||||||
| 6009 | case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y) | ||||||||
| 6010 | case FCmpInst::FCMP_ULT: // True if unordered or less than | ||||||||
| 6011 | case FCmpInst::FCMP_UGT: // True if unordered or greater than | ||||||||
| 6012 | case FCmpInst::FCMP_UNE: // True if unordered or not equal | ||||||||
| 6013 | // Canonicalize these to be 'fcmp uno %X, 0.0'. | ||||||||
| 6014 | I.setPredicate(FCmpInst::FCMP_UNO); | ||||||||
| 6015 | I.setOperand(1, Constant::getNullValue(OpType)); | ||||||||
| 6016 | return &I; | ||||||||
| 6017 | |||||||||
| 6018 | case FCmpInst::FCMP_ORD: // True if ordered (no nans) | ||||||||
| 6019 | case FCmpInst::FCMP_OEQ: // True if ordered and equal | ||||||||
| 6020 | case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal | ||||||||
| 6021 | case FCmpInst::FCMP_OLE: // True if ordered and less than or equal | ||||||||
| 6022 | // Canonicalize these to be 'fcmp ord %X, 0.0'. | ||||||||
| 6023 | I.setPredicate(FCmpInst::FCMP_ORD); | ||||||||
| 6024 | I.setOperand(1, Constant::getNullValue(OpType)); | ||||||||
| 6025 | return &I; | ||||||||
| 6026 | } | ||||||||
| 6027 | } | ||||||||
| 6028 | |||||||||
| 6029 | // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand, | ||||||||
| 6030 | // then canonicalize the operand to 0.0. | ||||||||
| 6031 | if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { | ||||||||
| 6032 | if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI)) | ||||||||
| 6033 | return replaceOperand(I, 0, ConstantFP::getNullValue(OpType)); | ||||||||
| 6034 | |||||||||
| 6035 | if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI)) | ||||||||
| 6036 | return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); | ||||||||
| 6037 | } | ||||||||
| 6038 | |||||||||
| 6039 | // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y | ||||||||
| 6040 | Value *X, *Y; | ||||||||
| 6041 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) | ||||||||
| 6042 | return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I); | ||||||||
| 6043 | |||||||||
| 6044 | // Test if the FCmpInst instruction is used exclusively by a select as | ||||||||
| 6045 | // part of a minimum or maximum operation. If so, refrain from doing | ||||||||
| 6046 | // any other folding. This helps out other analyses which understand | ||||||||
| 6047 | // non-obfuscated minimum and maximum idioms, such as ScalarEvolution | ||||||||
| 6048 | // and CodeGen. And in this case, at least one of the comparison | ||||||||
| 6049 | // operands has at least one user besides the compare (the select), | ||||||||
| 6050 | // which would often largely negate the benefit of folding anyway. | ||||||||
| 6051 | if (I.hasOneUse()) | ||||||||
| 6052 | if (SelectInst *SI = dyn_cast<SelectInst>(I.user_back())) { | ||||||||
| 6053 | Value *A, *B; | ||||||||
| 6054 | SelectPatternResult SPR = matchSelectPattern(SI, A, B); | ||||||||
| 6055 | if (SPR.Flavor != SPF_UNKNOWN) | ||||||||
| 6056 | return nullptr; | ||||||||
| 6057 | } | ||||||||
| 6058 | |||||||||
| 6059 | // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0: | ||||||||
| 6060 | // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0 | ||||||||
| 6061 | if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP())) | ||||||||
| 6062 | return replaceOperand(I, 1, ConstantFP::getNullValue(OpType)); | ||||||||
| 6063 | |||||||||
| 6064 | // Handle fcmp with instruction LHS and constant RHS. | ||||||||
| 6065 | Instruction *LHSI; | ||||||||
| 6066 | Constant *RHSC; | ||||||||
| 6067 | if (match(Op0, m_Instruction(LHSI)) && match(Op1, m_Constant(RHSC))) { | ||||||||
| 6068 | switch (LHSI->getOpcode()) { | ||||||||
| 6069 | case Instruction::PHI: | ||||||||
| 6070 | // Only fold fcmp into the PHI if the phi and fcmp are in the same | ||||||||
| 6071 | // block. If in the same block, we're encouraging jump threading. If | ||||||||
| 6072 | // not, we are just pessimizing the code by making an i1 phi. | ||||||||
| 6073 | if (LHSI->getParent() == I.getParent()) | ||||||||
| 6074 | if (Instruction *NV = foldOpIntoPhi(I, cast<PHINode>(LHSI))) | ||||||||
| 6075 | return NV; | ||||||||
| 6076 | break; | ||||||||
| 6077 | case Instruction::SIToFP: | ||||||||
| 6078 | case Instruction::UIToFP: | ||||||||
| 6079 | if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC)) | ||||||||
| 6080 | return NV; | ||||||||
| 6081 | break; | ||||||||
| 6082 | case Instruction::FDiv: | ||||||||
| 6083 | if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC)) | ||||||||
| 6084 | return NV; | ||||||||
| 6085 | break; | ||||||||
| 6086 | case Instruction::Load: | ||||||||
| 6087 | if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) | ||||||||
| 6088 | if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) | ||||||||
| 6089 | if (GV->isConstant() && GV->hasDefinitiveInitializer() && | ||||||||
| 6090 | !cast<LoadInst>(LHSI)->isVolatile()) | ||||||||
| 6091 | if (Instruction *Res = foldCmpLoadFromIndexedGlobal(GEP, GV, I)) | ||||||||
| 6092 | return Res; | ||||||||
| 6093 | break; | ||||||||
| 6094 | } | ||||||||
| 6095 | } | ||||||||
| 6096 | |||||||||
| 6097 | if (Instruction *R = foldFabsWithFcmpZero(I)) | ||||||||
| 6098 | return R; | ||||||||
| 6099 | |||||||||
| 6100 | if (match(Op0, m_FNeg(m_Value(X)))) { | ||||||||
| 6101 | // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C | ||||||||
| 6102 | Constant *C; | ||||||||
| 6103 | if (match(Op1, m_Constant(C))) { | ||||||||
| 6104 | Constant *NegC = ConstantExpr::getFNeg(C); | ||||||||
| 6105 | return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I); | ||||||||
| 6106 | } | ||||||||
| 6107 | } | ||||||||
| 6108 | |||||||||
| 6109 | if (match(Op0, m_FPExt(m_Value(X)))) { | ||||||||
| 6110 | // fcmp (fpext X), (fpext Y) -> fcmp X, Y | ||||||||
| 6111 | if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType()) | ||||||||
| 6112 | return new FCmpInst(Pred, X, Y, "", &I); | ||||||||
| 6113 | |||||||||
| 6114 | // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless | ||||||||
| 6115 | const APFloat *C; | ||||||||
| 6116 | if (match(Op1, m_APFloat(C))) { | ||||||||
| 6117 | const fltSemantics &FPSem = | ||||||||
| 6118 | X->getType()->getScalarType()->getFltSemantics(); | ||||||||
| 6119 | bool Lossy; | ||||||||
| 6120 | APFloat TruncC = *C; | ||||||||
| 6121 | TruncC.convert(FPSem, APFloat::rmNearestTiesToEven, &Lossy); | ||||||||
| 6122 | |||||||||
| 6123 | // Avoid lossy conversions and denormals. | ||||||||
| 6124 | // Zero is a special case that's OK to convert. | ||||||||
| 6125 | APFloat Fabs = TruncC; | ||||||||
| 6126 | Fabs.clearSign(); | ||||||||
| 6127 | if (!Lossy && | ||||||||
| 6128 | ((Fabs.compare(APFloat::getSmallestNormalized(FPSem)) != | ||||||||
| 6129 | APFloat::cmpLessThan) || Fabs.isZero())) { | ||||||||
| 6130 | Constant *NewC = ConstantFP::get(X->getType(), TruncC); | ||||||||
| 6131 | return new FCmpInst(Pred, X, NewC, "", &I); | ||||||||
| 6132 | } | ||||||||
| 6133 | } | ||||||||
| 6134 | } | ||||||||
| 6135 | |||||||||
| 6136 | if (I.getType()->isVectorTy()) | ||||||||
| 6137 | if (Instruction *Res = foldVectorCmp(I, Builder)) | ||||||||
| 6138 | return Res; | ||||||||
| 6139 | |||||||||
| 6140 | return Changed ? &I : nullptr; | ||||||||
| 6141 | } |
| 1 | //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file provides a simple and efficient mechanism for performing general |
| 10 | // tree-based pattern matches on the LLVM IR. The power of these routines is |
| 11 | // that it allows you to write concise patterns that are expressive and easy to |
| 12 | // understand. The other major advantage of this is that it allows you to |
| 13 | // trivially capture/bind elements in the pattern to variables. For example, |
| 14 | // you can do something like this: |
| 15 | // |
| 16 | // Value *Exp = ... |
| 17 | // Value *X, *Y; ConstantInt *C1, *C2; // (X & C1) | (Y & C2) |
| 18 | // if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)), |
| 19 | // m_And(m_Value(Y), m_ConstantInt(C2))))) { |
| 20 | // ... Pattern is matched and variables are bound ... |
| 21 | // } |
| 22 | // |
| 23 | // This is primarily useful to things like the instruction combiner, but can |
| 24 | // also be useful for static analysis tools or code generators. |
| 25 | // |
| 26 | //===----------------------------------------------------------------------===// |
| 27 | |
| 28 | #ifndef LLVM_IR_PATTERNMATCH_H |
| 29 | #define LLVM_IR_PATTERNMATCH_H |
| 30 | |
| 31 | #include "llvm/ADT/APFloat.h" |
| 32 | #include "llvm/ADT/APInt.h" |
| 33 | #include "llvm/IR/Constant.h" |
| 34 | #include "llvm/IR/Constants.h" |
| 35 | #include "llvm/IR/DataLayout.h" |
| 36 | #include "llvm/IR/InstrTypes.h" |
| 37 | #include "llvm/IR/Instruction.h" |
| 38 | #include "llvm/IR/Instructions.h" |
| 39 | #include "llvm/IR/IntrinsicInst.h" |
| 40 | #include "llvm/IR/Intrinsics.h" |
| 41 | #include "llvm/IR/Operator.h" |
| 42 | #include "llvm/IR/Value.h" |
| 43 | #include "llvm/Support/Casting.h" |
| 44 | #include <cstdint> |
| 45 | |
| 46 | namespace llvm { |
| 47 | namespace PatternMatch { |
| 48 | |
| 49 | template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) { |
| 50 | return const_cast<Pattern &>(P).match(V); |
| 51 | } |
| 52 | |
| 53 | template <typename SubPattern_t> struct OneUse_match { |
| 54 | SubPattern_t SubPattern; |
| 55 | |
| 56 | OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} |
| 57 | |
| 58 | template <typename OpTy> bool match(OpTy *V) { |
| 59 | return V->hasOneUse() && SubPattern.match(V); |
| 60 | } |
| 61 | }; |
| 62 | |
| 63 | template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) { |
| 64 | return SubPattern; |
| 65 | } |
| 66 | |
| 67 | template <typename Class> struct class_match { |
| 68 | template <typename ITy> bool match(ITy *V) { return isa<Class>(V); } |
| 69 | }; |
| 70 | |
| 71 | /// Match an arbitrary value and ignore it. |
| 72 | inline class_match<Value> m_Value() { return class_match<Value>(); } |
| 73 | |
| 74 | /// Match an arbitrary binary operation and ignore it. |
| 75 | inline class_match<BinaryOperator> m_BinOp() { |
| 76 | return class_match<BinaryOperator>(); |
| 77 | } |
| 78 | |
| 79 | /// Matches any compare instruction and ignore it. |
| 80 | inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); } |
| 81 | |
| 82 | /// Match an arbitrary ConstantInt and ignore it. |
| 83 | inline class_match<ConstantInt> m_ConstantInt() { |
| 84 | return class_match<ConstantInt>(); |
| 85 | } |
| 86 | |
| 87 | /// Match an arbitrary undef constant. |
| 88 | inline class_match<UndefValue> m_Undef() { return class_match<UndefValue>(); } |
| 89 | |
| 90 | /// Match an arbitrary Constant and ignore it. |
| 91 | inline class_match<Constant> m_Constant() { return class_match<Constant>(); } |
| 92 | |
| 93 | /// Match an arbitrary basic block value and ignore it. |
| 94 | inline class_match<BasicBlock> m_BasicBlock() { |
| 95 | return class_match<BasicBlock>(); |
| 96 | } |
| 97 | |
| 98 | /// Inverting matcher |
| 99 | template <typename Ty> struct match_unless { |
| 100 | Ty M; |
| 101 | |
| 102 | match_unless(const Ty &Matcher) : M(Matcher) {} |
| 103 | |
| 104 | template <typename ITy> bool match(ITy *V) { return !M.match(V); } |
| 105 | }; |
| 106 | |
| 107 | /// Match if the inner matcher does *NOT* match. |
| 108 | template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) { |
| 109 | return match_unless<Ty>(M); |
| 110 | } |
| 111 | |
| 112 | /// Matching combinators |
| 113 | template <typename LTy, typename RTy> struct match_combine_or { |
| 114 | LTy L; |
| 115 | RTy R; |
| 116 | |
| 117 | match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} |
| 118 | |
| 119 | template <typename ITy> bool match(ITy *V) { |
| 120 | if (L.match(V)) |
| 121 | return true; |
| 122 | if (R.match(V)) |
| 123 | return true; |
| 124 | return false; |
| 125 | } |
| 126 | }; |
| 127 | |
| 128 | template <typename LTy, typename RTy> struct match_combine_and { |
| 129 | LTy L; |
| 130 | RTy R; |
| 131 | |
| 132 | match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} |
| 133 | |
| 134 | template <typename ITy> bool match(ITy *V) { |
| 135 | if (L.match(V)) |
| 136 | if (R.match(V)) |
| 137 | return true; |
| 138 | return false; |
| 139 | } |
| 140 | }; |
| 141 | |
| 142 | /// Combine two pattern matchers matching L || R |
| 143 | template <typename LTy, typename RTy> |
| 144 | inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) { |
| 145 | return match_combine_or<LTy, RTy>(L, R); |
| 146 | } |
| 147 | |
| 148 | /// Combine two pattern matchers matching L && R |
| 149 | template <typename LTy, typename RTy> |
| 150 | inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) { |
| 151 | return match_combine_and<LTy, RTy>(L, R); |
| 152 | } |
| 153 | |
| 154 | struct apint_match { |
| 155 | const APInt *&Res; |
| 156 | bool AllowUndef; |
| 157 | |
| 158 | apint_match(const APInt *&Res, bool AllowUndef) |
| 159 | : Res(Res), AllowUndef(AllowUndef) {} |
| 160 | |
| 161 | template <typename ITy> bool match(ITy *V) { |
| 162 | if (auto *CI = dyn_cast<ConstantInt>(V)) { |
| 163 | Res = &CI->getValue(); |
| 164 | return true; |
| 165 | } |
| 166 | if (V->getType()->isVectorTy()) |
| 167 | if (const auto *C = dyn_cast<Constant>(V)) |
| 168 | if (auto *CI = dyn_cast_or_null<ConstantInt>( |
| 169 | C->getSplatValue(AllowUndef))) { |
| 170 | Res = &CI->getValue(); |
| 171 | return true; |
| 172 | } |
| 173 | return false; |
| 174 | } |
| 175 | }; |
| 176 | // Either constexpr if or renaming ConstantFP::getValueAPF to |
| 177 | // ConstantFP::getValue is needed to do it via single template |
| 178 | // function for both apint/apfloat. |
| 179 | struct apfloat_match { |
| 180 | const APFloat *&Res; |
| 181 | bool AllowUndef; |
| 182 | |
| 183 | apfloat_match(const APFloat *&Res, bool AllowUndef) |
| 184 | : Res(Res), AllowUndef(AllowUndef) {} |
| 185 | |
| 186 | template <typename ITy> bool match(ITy *V) { |
| 187 | if (auto *CI = dyn_cast<ConstantFP>(V)) { |
| 188 | Res = &CI->getValueAPF(); |
| 189 | return true; |
| 190 | } |
| 191 | if (V->getType()->isVectorTy()) |
| 192 | if (const auto *C = dyn_cast<Constant>(V)) |
| 193 | if (auto *CI = dyn_cast_or_null<ConstantFP>( |
| 194 | C->getSplatValue(AllowUndef))) { |
| 195 | Res = &CI->getValueAPF(); |
| 196 | return true; |
| 197 | } |
| 198 | return false; |
| 199 | } |
| 200 | }; |
| 201 | |
| 202 | /// Match a ConstantInt or splatted ConstantVector, binding the |
| 203 | /// specified pointer to the contained APInt. |
| 204 | inline apint_match m_APInt(const APInt *&Res) { |
| 205 | // Forbid undefs by default to maintain previous behavior. |
| 206 | return apint_match(Res, /* AllowUndef */ false); |
| 207 | } |
| 208 | |
| 209 | /// Match APInt while allowing undefs in splat vector constants. |
| 210 | inline apint_match m_APIntAllowUndef(const APInt *&Res) { |
| 211 | return apint_match(Res, /* AllowUndef */ true); |
| 212 | } |
| 213 | |
| 214 | /// Match APInt while forbidding undefs in splat vector constants. |
| 215 | inline apint_match m_APIntForbidUndef(const APInt *&Res) { |
| 216 | return apint_match(Res, /* AllowUndef */ false); |
| 217 | } |
| 218 | |
| 219 | /// Match a ConstantFP or splatted ConstantVector, binding the |
| 220 | /// specified pointer to the contained APFloat. |
| 221 | inline apfloat_match m_APFloat(const APFloat *&Res) { |
| 222 | // Forbid undefs by default to maintain previous behavior. |
| 223 | return apfloat_match(Res, /* AllowUndef */ false); |
| 224 | } |
| 225 | |
| 226 | /// Match APFloat while allowing undefs in splat vector constants. |
| 227 | inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) { |
| 228 | return apfloat_match(Res, /* AllowUndef */ true); |
| 229 | } |
| 230 | |
| 231 | /// Match APFloat while forbidding undefs in splat vector constants. |
| 232 | inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) { |
| 233 | return apfloat_match(Res, /* AllowUndef */ false); |
| 234 | } |
| 235 | |
| 236 | template <int64_t Val> struct constantint_match { |
| 237 | template <typename ITy> bool match(ITy *V) { |
| 238 | if (const auto *CI = dyn_cast<ConstantInt>(V)) { |
| 239 | const APInt &CIV = CI->getValue(); |
| 240 | if (Val >= 0) |
| 241 | return CIV == static_cast<uint64_t>(Val); |
| 242 | // If Val is negative, and CI is shorter than it, truncate to the right |
| 243 | // number of bits. If it is larger, then we have to sign extend. Just |
| 244 | // compare their negated values. |
| 245 | return -CIV == -Val; |
| 246 | } |
| 247 | return false; |
| 248 | } |
| 249 | }; |
| 250 | |
| 251 | /// Match a ConstantInt with a specific value. |
| 252 | template <int64_t Val> inline constantint_match<Val> m_ConstantInt() { |
| 253 | return constantint_match<Val>(); |
| 254 | } |
| 255 | |
| 256 | /// This helper class is used to match scalar and vector integer constants that |
| 257 | /// satisfy a specified predicate. |
| 258 | /// For vector constants, undefined elements are ignored. |
| 259 | template <typename Predicate> struct cst_pred_ty : public Predicate { |
| 260 | template <typename ITy> bool match(ITy *V) { |
| 261 | if (const auto *CI = dyn_cast<ConstantInt>(V)) |
| 262 | return this->isValue(CI->getValue()); |
| 263 | if (V->getType()->isVectorTy()) { |
| 264 | if (const auto *C = dyn_cast<Constant>(V)) { |
| 265 | if (const auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue())) |
| 266 | return this->isValue(CI->getValue()); |
| 267 | |
| 268 | // Non-splat vector constant: check each element for a match. |
| 269 | unsigned NumElts = V->getType()->getVectorNumElements(); |
| 270 | assert(NumElts != 0 && "Constant vector with no elements?")((NumElts != 0 && "Constant vector with no elements?" ) ? static_cast<void> (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/include/llvm/IR/PatternMatch.h" , 270, __PRETTY_FUNCTION__)); |
| 271 | bool HasNonUndefElements = false; |
| 272 | for (unsigned i = 0; i != NumElts; ++i) { |
| 273 | Constant *Elt = C->getAggregateElement(i); |
| 274 | if (!Elt) |
| 275 | return false; |
| 276 | if (isa<UndefValue>(Elt)) |
| 277 | continue; |
| 278 | auto *CI = dyn_cast<ConstantInt>(Elt); |
| 279 | if (!CI || !this->isValue(CI->getValue())) |
| 280 | return false; |
| 281 | HasNonUndefElements = true; |
| 282 | } |
| 283 | return HasNonUndefElements; |
| 284 | } |
| 285 | } |
| 286 | return false; |
| 287 | } |
| 288 | }; |
| 289 | |
| 290 | /// This helper class is used to match scalar and vector constants that |
| 291 | /// satisfy a specified predicate, and bind them to an APInt. |
| 292 | template <typename Predicate> struct api_pred_ty : public Predicate { |
| 293 | const APInt *&Res; |
| 294 | |
| 295 | api_pred_ty(const APInt *&R) : Res(R) {} |
| 296 | |
| 297 | template <typename ITy> bool match(ITy *V) { |
| 298 | if (const auto *CI = dyn_cast<ConstantInt>(V)) |
| 299 | if (this->isValue(CI->getValue())) { |
| 300 | Res = &CI->getValue(); |
| 301 | return true; |
| 302 | } |
| 303 | if (V->getType()->isVectorTy()) |
| 304 | if (const auto *C = dyn_cast<Constant>(V)) |
| 305 | if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue())) |
| 306 | if (this->isValue(CI->getValue())) { |
| 307 | Res = &CI->getValue(); |
| 308 | return true; |
| 309 | } |
| 310 | |
| 311 | return false; |
| 312 | } |
| 313 | }; |
| 314 | |
| 315 | /// This helper class is used to match scalar and vector floating-point |
| 316 | /// constants that satisfy a specified predicate. |
| 317 | /// For vector constants, undefined elements are ignored. |
| 318 | template <typename Predicate> struct cstfp_pred_ty : public Predicate { |
| 319 | template <typename ITy> bool match(ITy *V) { |
| 320 | if (const auto *CF = dyn_cast<ConstantFP>(V)) |
| 321 | return this->isValue(CF->getValueAPF()); |
| 322 | if (V->getType()->isVectorTy()) { |
| 323 | if (const auto *C = dyn_cast<Constant>(V)) { |
| 324 | if (const auto *CF = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) |
| 325 | return this->isValue(CF->getValueAPF()); |
| 326 | |
| 327 | // Non-splat vector constant: check each element for a match. |
| 328 | unsigned NumElts = V->getType()->getVectorNumElements(); |
| 329 | assert(NumElts != 0 && "Constant vector with no elements?")((NumElts != 0 && "Constant vector with no elements?" ) ? static_cast<void> (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\"" , "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/include/llvm/IR/PatternMatch.h" , 329, __PRETTY_FUNCTION__)); |
| 330 | bool HasNonUndefElements = false; |
| 331 | for (unsigned i = 0; i != NumElts; ++i) { |
| 332 | Constant *Elt = C->getAggregateElement(i); |
| 333 | if (!Elt) |
| 334 | return false; |
| 335 | if (isa<UndefValue>(Elt)) |
| 336 | continue; |
| 337 | auto *CF = dyn_cast<ConstantFP>(Elt); |
| 338 | if (!CF || !this->isValue(CF->getValueAPF())) |
| 339 | return false; |
| 340 | HasNonUndefElements = true; |
| 341 | } |
| 342 | return HasNonUndefElements; |
| 343 | } |
| 344 | } |
| 345 | return false; |
| 346 | } |
| 347 | }; |
| 348 | |
| 349 | /////////////////////////////////////////////////////////////////////////////// |
| 350 | // |
| 351 | // Encapsulate constant value queries for use in templated predicate matchers. |
| 352 | // This allows checking if constants match using compound predicates and works |
| 353 | // with vector constants, possibly with relaxed constraints. For example, ignore |
| 354 | // undef values. |
| 355 | // |
| 356 | /////////////////////////////////////////////////////////////////////////////// |
| 357 | |
| 358 | struct is_any_apint { |
| 359 | bool isValue(const APInt &C) { return true; } |
| 360 | }; |
| 361 | /// Match an integer or vector with any integral constant. |
| 362 | /// For vectors, this includes constants with undefined elements. |
| 363 | inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { |
| 364 | return cst_pred_ty<is_any_apint>(); |
| 365 | } |
| 366 | |
| 367 | struct is_all_ones { |
| 368 | bool isValue(const APInt &C) { return C.isAllOnesValue(); } |
| 369 | }; |
| 370 | /// Match an integer or vector with all bits set. |
| 371 | /// For vectors, this includes constants with undefined elements. |
| 372 | inline cst_pred_ty<is_all_ones> m_AllOnes() { |
| 373 | return cst_pred_ty<is_all_ones>(); |
| 374 | } |
| 375 | |
| 376 | struct is_maxsignedvalue { |
| 377 | bool isValue(const APInt &C) { return C.isMaxSignedValue(); } |
| 378 | }; |
| 379 | /// Match an integer or vector with values having all bits except for the high |
| 380 | /// bit set (0x7f...). |
| 381 | /// For vectors, this includes constants with undefined elements. |
| 382 | inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() { |
| 383 | return cst_pred_ty<is_maxsignedvalue>(); |
| 384 | } |
| 385 | inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) { |
| 386 | return V; |
| 387 | } |
| 388 | |
| 389 | struct is_negative { |
| 390 | bool isValue(const APInt &C) { return C.isNegative(); } |
| 391 | }; |
| 392 | /// Match an integer or vector of negative values. |
| 393 | /// For vectors, this includes constants with undefined elements. |
| 394 | inline cst_pred_ty<is_negative> m_Negative() { |
| 395 | return cst_pred_ty<is_negative>(); |
| 396 | } |
| 397 | inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { |
| 398 | return V; |
| 399 | } |
| 400 | |
| 401 | struct is_nonnegative { |
| 402 | bool isValue(const APInt &C) { return C.isNonNegative(); } |
| 403 | }; |
| 404 | /// Match an integer or vector of non-negative values. |
| 405 | /// For vectors, this includes constants with undefined elements. |
| 406 | inline cst_pred_ty<is_nonnegative> m_NonNegative() { |
| 407 | return cst_pred_ty<is_nonnegative>(); |
| 408 | } |
| 409 | inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { |
| 410 | return V; |
| 411 | } |
| 412 | |
| 413 | struct is_strictlypositive { |
| 414 | bool isValue(const APInt &C) { return C.isStrictlyPositive(); } |
| 415 | }; |
| 416 | /// Match an integer or vector of strictly positive values. |
| 417 | /// For vectors, this includes constants with undefined elements. |
| 418 | inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() { |
| 419 | return cst_pred_ty<is_strictlypositive>(); |
| 420 | } |
| 421 | inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) { |
| 422 | return V; |
| 423 | } |
| 424 | |
| 425 | struct is_nonpositive { |
| 426 | bool isValue(const APInt &C) { return C.isNonPositive(); } |
| 427 | }; |
| 428 | /// Match an integer or vector of non-positive values. |
| 429 | /// For vectors, this includes constants with undefined elements. |
| 430 | inline cst_pred_ty<is_nonpositive> m_NonPositive() { |
| 431 | return cst_pred_ty<is_nonpositive>(); |
| 432 | } |
| 433 | inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } |
| 434 | |
| 435 | struct is_one { |
| 436 | bool isValue(const APInt &C) { return C.isOneValue(); } |
| 437 | }; |
| 438 | /// Match an integer 1 or a vector with all elements equal to 1. |
| 439 | /// For vectors, this includes constants with undefined elements. |
| 440 | inline cst_pred_ty<is_one> m_One() { |
| 441 | return cst_pred_ty<is_one>(); |
| 442 | } |
| 443 | |
| 444 | struct is_zero_int { |
| 445 | bool isValue(const APInt &C) { return C.isNullValue(); } |
| 446 | }; |
| 447 | /// Match an integer 0 or a vector with all elements equal to 0. |
| 448 | /// For vectors, this includes constants with undefined elements. |
| 449 | inline cst_pred_ty<is_zero_int> m_ZeroInt() { |
| 450 | return cst_pred_ty<is_zero_int>(); |
| 451 | } |
| 452 | |
| 453 | struct is_zero { |
| 454 | template <typename ITy> bool match(ITy *V) { |
| 455 | auto *C = dyn_cast<Constant>(V); |
| 456 | return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C)); |
| 457 | } |
| 458 | }; |
| 459 | /// Match any null constant or a vector with all elements equal to 0. |
| 460 | /// For vectors, this includes constants with undefined elements. |
| 461 | inline is_zero m_Zero() { |
| 462 | return is_zero(); |
| 463 | } |
| 464 | |
| 465 | struct is_power2 { |
| 466 | bool isValue(const APInt &C) { return C.isPowerOf2(); } |
| 467 | }; |
| 468 | /// Match an integer or vector power-of-2. |
| 469 | /// For vectors, this includes constants with undefined elements. |
| 470 | inline cst_pred_ty<is_power2> m_Power2() { |
| 471 | return cst_pred_ty<is_power2>(); |
| 472 | } |
| 473 | inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { |
| 474 | return V; |
| 475 | } |
| 476 | |
| 477 | struct is_negated_power2 { |
| 478 | bool isValue(const APInt &C) { return (-C).isPowerOf2(); } |
| 479 | }; |
| 480 | /// Match a integer or vector negated power-of-2. |
| 481 | /// For vectors, this includes constants with undefined elements. |
| 482 | inline cst_pred_ty<is_negated_power2> m_NegatedPower2() { |
| 483 | return cst_pred_ty<is_negated_power2>(); |
| 484 | } |
| 485 | inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) { |
| 486 | return V; |
| 487 | } |
| 488 | |
| 489 | struct is_power2_or_zero { |
| 490 | bool isValue(const APInt &C) { return !C || C.isPowerOf2(); } |
| 491 | }; |
| 492 | /// Match an integer or vector of 0 or power-of-2 values. |
| 493 | /// For vectors, this includes constants with undefined elements. |
| 494 | inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() { |
| 495 | return cst_pred_ty<is_power2_or_zero>(); |
| 496 | } |
| 497 | inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) { |
| 498 | return V; |
| 499 | } |
| 500 | |
| 501 | struct is_sign_mask { |
| 502 | bool isValue(const APInt &C) { return C.isSignMask(); } |
| 503 | }; |
| 504 | /// Match an integer or vector with only the sign bit(s) set. |
| 505 | /// For vectors, this includes constants with undefined elements. |
| 506 | inline cst_pred_ty<is_sign_mask> m_SignMask() { |
| 507 | return cst_pred_ty<is_sign_mask>(); |
| 508 | } |
| 509 | |
| 510 | struct is_lowbit_mask { |
| 511 | bool isValue(const APInt &C) { return C.isMask(); } |
| 512 | }; |
| 513 | /// Match an integer or vector with only the low bit(s) set. |
| 514 | /// For vectors, this includes constants with undefined elements. |
| 515 | inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { |
| 516 | return cst_pred_ty<is_lowbit_mask>(); |
| 517 | } |
| 518 | |
| 519 | struct icmp_pred_with_threshold { |
| 520 | ICmpInst::Predicate Pred; |
| 521 | const APInt *Thr; |
| 522 | bool isValue(const APInt &C) { |
| 523 | switch (Pred) { |
| 524 | case ICmpInst::Predicate::ICMP_EQ: |
| 525 | return C.eq(*Thr); |
| 526 | case ICmpInst::Predicate::ICMP_NE: |
| 527 | return C.ne(*Thr); |
| 528 | case ICmpInst::Predicate::ICMP_UGT: |
| 529 | return C.ugt(*Thr); |
| 530 | case ICmpInst::Predicate::ICMP_UGE: |
| 531 | return C.uge(*Thr); |
| 532 | case ICmpInst::Predicate::ICMP_ULT: |
| 533 | return C.ult(*Thr); |
| 534 | case ICmpInst::Predicate::ICMP_ULE: |
| 535 | return C.ule(*Thr); |
| 536 | case ICmpInst::Predicate::ICMP_SGT: |
| 537 | return C.sgt(*Thr); |
| 538 | case ICmpInst::Predicate::ICMP_SGE: |
| 539 | return C.sge(*Thr); |
| 540 | case ICmpInst::Predicate::ICMP_SLT: |
| 541 | return C.slt(*Thr); |
| 542 | case ICmpInst::Predicate::ICMP_SLE: |
| 543 | return C.sle(*Thr); |
| 544 | default: |
| 545 | llvm_unreachable("Unhandled ICmp predicate")::llvm::llvm_unreachable_internal("Unhandled ICmp predicate", "/build/llvm-toolchain-snapshot-11~++20200225111114+dcd89b3de6d/llvm/include/llvm/IR/PatternMatch.h" , 545); |
| 546 | } |
| 547 | } |
| 548 | }; |
| 549 | /// Match an integer or vector with every element comparing 'pred' (eg/ne/...) |
| 550 | /// to Threshold. For vectors, this includes constants with undefined elements. |
| 551 | inline cst_pred_ty<icmp_pred_with_threshold> |
| 552 | m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) { |
| 553 | cst_pred_ty<icmp_pred_with_threshold> P; |
| 554 | P.Pred = Predicate; |
| 555 | P.Thr = &Threshold; |
| 556 | return P; |
| 557 | } |
| 558 | |
| 559 | struct is_nan { |
| 560 | bool isValue(const APFloat &C) { return C.isNaN(); } |
| 561 | }; |
| 562 | /// Match an arbitrary NaN constant. This includes quiet and signalling nans. |
| 563 | /// For vectors, this includes constants with undefined elements. |
| 564 | inline cstfp_pred_ty<is_nan> m_NaN() { |
| 565 | return cstfp_pred_ty<is_nan>(); |
| 566 | } |
| 567 | |
| 568 | struct is_any_zero_fp { |
| 569 | bool isValue(const APFloat &C) { return C.isZero(); } |
| 570 | }; |
| 571 | /// Match a floating-point negative zero or positive zero. |
| 572 | /// For vectors, this includes constants with undefined elements. |
| 573 | inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() { |
| 574 | return cstfp_pred_ty<is_any_zero_fp>(); |
| 575 | } |
| 576 | |
| 577 | struct is_pos_zero_fp { |
| 578 | bool isValue(const APFloat &C) { return C.isPosZero(); } |
| 579 | }; |
| 580 | /// Match a floating-point positive zero. |
| 581 | /// For vectors, this includes constants with undefined elements. |
| 582 | inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() { |
| 583 | return cstfp_pred_ty<is_pos_zero_fp>(); |
| 584 | } |
| 585 | |
| 586 | struct is_neg_zero_fp { |
| 587 | bool isValue(const APFloat &C) { return C.isNegZero(); } |
| 588 | }; |
| 589 | /// Match a floating-point negative zero. |
| 590 | /// For vectors, this includes constants with undefined elements. |
| 591 | inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() { |
| 592 | return cstfp_pred_ty<is_neg_zero_fp>(); |
| 593 | } |
| 594 | |
| 595 | /////////////////////////////////////////////////////////////////////////////// |
| 596 | |
| 597 | template <typename Class> struct bind_ty { |
| 598 | Class *&VR; |
| 599 | |
| 600 | bind_ty(Class *&V) : VR(V) {} |
| 601 | |
| 602 | template <typename ITy> bool match(ITy *V) { |
| 603 | if (auto *CV = dyn_cast<Class>(V)) { |
| 604 | VR = CV; |
| 605 | return true; |
| 606 | } |
| 607 | return false; |
| 608 | } |
| 609 | }; |
| 610 | |
| 611 | /// Match a value, capturing it if we match. |
| 612 | inline bind_ty<Value> m_Value(Value *&V) { return V; } |
| 613 | inline bind_ty<const Value> m_Value(const Value *&V) { return V; } |
| 614 | |
| 615 | /// Match an instruction, capturing it if we match. |
| 616 | inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } |
| 617 | /// Match a binary operator, capturing it if we match. |
| 618 | inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } |
| 619 | /// Match a with overflow intrinsic, capturing it if we match. |
| 620 | inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { return I; } |
| 621 | |
| 622 | /// Match a ConstantInt, capturing the value if we match. |
| 623 | inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; } |
| 624 | |
| 625 | /// Match a Constant, capturing the value if we match. |
| 626 | inline bind_ty<Constant> m_Constant(Constant *&C) { return C; } |
| 627 | |
| 628 | /// Match a ConstantFP, capturing the value if we match. |
| 629 | inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; } |
| 630 | |
| 631 | /// Match a basic block value, capturing it if we match. |
| 632 | inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; } |
| 633 | inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) { |
| 634 | return V; |
| 635 | } |
| 636 | |
| 637 | /// Match a specified Value*. |
| 638 | struct specificval_ty { |
| 639 | const Value *Val; |
| 640 | |
| 641 | specificval_ty(const Value *V) : Val(V) {} |
| 642 | |
| 643 | template <typename ITy> bool match(ITy *V) { return V == Val; } |
| 644 | }; |
| 645 | |
| 646 | /// Match if we have a specific specified value. |
| 647 | inline specificval_ty m_Specific(const Value *V) { return V; } |
| 648 | |
| 649 | /// Stores a reference to the Value *, not the Value * itself, |
| 650 | /// thus can be used in commutative matchers. |
| 651 | template <typename Class> struct deferredval_ty { |
| 652 | Class *const &Val; |
| 653 | |
| 654 | deferredval_ty(Class *const &V) : Val(V) {} |
| 655 | |
| 656 | template <typename ITy> bool match(ITy *const V) { return V == Val; } |
| 657 | }; |
| 658 | |
| 659 | /// A commutative-friendly version of m_Specific(). |
| 660 | inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; } |
| 661 | inline deferredval_ty<const Value> m_Deferred(const Value *const &V) { |
| 662 | return V; |
| 663 | } |
| 664 | |
| 665 | /// Match a specified floating point value or vector of all elements of |
| 666 | /// that value. |
| 667 | struct specific_fpval { |
| 668 | double Val; |
| 669 | |
| 670 | specific_fpval(double V) : Val(V) {} |
| 671 | |
| 672 | template <typename ITy> bool match(ITy *V) { |
| 673 | if (const auto *CFP = dyn_cast<ConstantFP>(V)) |
| 674 | return CFP->isExactlyValue(Val); |
| 675 | if (V->getType()->isVectorTy()) |
| 676 | if (const auto *C = dyn_cast<Constant>(V)) |
| 677 | if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) |
| 678 | return CFP->isExactlyValue(Val); |
| 679 | return false; |
| 680 | } |
| 681 | }; |
| 682 | |
| 683 | /// Match a specific floating point value or vector with all elements |
| 684 | /// equal to the value. |
| 685 | inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); } |
| 686 | |
| 687 | /// Match a float 1.0 or vector with all elements equal to 1.0. |
| 688 | inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); } |
| 689 | |
| 690 | struct bind_const_intval_ty { |
| 691 | uint64_t &VR; |
| 692 | |
| 693 | bind_const_intval_ty(uint64_t &V) : VR(V) {} |
| 694 | |
| 695 | template <typename ITy> bool match(ITy *V) { |
| 696 | if (const auto *CV = dyn_cast<ConstantInt>(V)) |
| 697 | if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) { |
| 698 | VR = CV->getZExtValue(); |
| 699 | return true; |
| 700 | } |
| 701 | return false; |
| 702 | } |
| 703 | }; |
| 704 | |
| 705 | /// Match a specified integer value or vector of all elements of that |
| 706 | /// value. |
| 707 | struct specific_intval { |
| 708 | APInt Val; |
| 709 | |
| 710 | specific_intval(APInt V) : Val(std::move(V)) {} |
| 711 | |
| 712 | template <typename ITy> bool match(ITy *V) { |
| 713 | const auto *CI = dyn_cast<ConstantInt>(V); |
| 714 | if (!CI && V->getType()->isVectorTy()) |
| 715 | if (const auto *C = dyn_cast<Constant>(V)) |
| 716 | CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue()); |
| 717 | |
| 718 | return CI && APInt::isSameValue(CI->getValue(), Val); |
| 719 | } |
| 720 | }; |
| 721 | |
| 722 | /// Match a specific integer value or vector with all elements equal to |
| 723 | /// the value. |
| 724 | inline specific_intval m_SpecificInt(APInt V) { |
| 725 | return specific_intval(std::move(V)); |
| 726 | } |
| 727 | |
| 728 | inline specific_intval m_SpecificInt(uint64_t V) { |
| 729 | return m_SpecificInt(APInt(64, V)); |
| 730 | } |
| 731 | |
| 732 | /// Match a ConstantInt and bind to its value. This does not match |
| 733 | /// ConstantInts wider than 64-bits. |
| 734 | inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } |
| 735 | |
| 736 | /// Match a specified basic block value. |
| 737 | struct specific_bbval { |
| 738 | BasicBlock *Val; |
| 739 | |
| 740 | specific_bbval(BasicBlock *Val) : Val(Val) {} |
| 741 | |
| 742 | template <typename ITy> bool match(ITy *V) { |
| 743 | const auto *BB = dyn_cast<BasicBlock>(V); |
| 744 | return BB && BB == Val; |
| 745 | } |
| 746 | }; |
| 747 | |
| 748 | /// Match a specific basic block value. |
| 749 | inline specific_bbval m_SpecificBB(BasicBlock *BB) { |
| 750 | return specific_bbval(BB); |
| 751 | } |
| 752 | |
| 753 | /// A commutative-friendly version of m_Specific(). |
| 754 | inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) { |
| 755 | return BB; |
| 756 | } |
| 757 | inline deferredval_ty<const BasicBlock> |
| 758 | m_Deferred(const BasicBlock *const &BB) { |
| 759 | return BB; |
| 760 | } |
| 761 | |
| 762 | //===----------------------------------------------------------------------===// |
| 763 | // Matcher for any binary operator. |
| 764 | // |
| 765 | template <typename LHS_t, typename RHS_t, bool Commutable = false> |
| 766 | struct AnyBinaryOp_match { |
| 767 | LHS_t L; |
| 768 | RHS_t R; |
| 769 | |
| 770 | // The evaluation order is always stable, regardless of Commutability. |
| 771 | // The LHS is always matched first. |
| 772 | AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
| 773 | |
| 774 | template <typename OpTy> bool match(OpTy *V) { |
| 775 | if (auto *I = dyn_cast<BinaryOperator>(V)) |
| 776 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || |
| 777 | (Commutable && L.match(I->getOperand(1)) && |
| 778 | R.match(I->getOperand(0))); |
| 779 | return false; |
| 780 | } |
| 781 | }; |
| 782 | |
| 783 | template <typename LHS, typename RHS> |
| 784 | inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) { |
| 785 | return AnyBinaryOp_match<LHS, RHS>(L, R); |
| 786 | } |
| 787 | |
| 788 | //===----------------------------------------------------------------------===// |
| 789 | // Matchers for specific binary operators. |
| 790 | // |
| 791 | |
| 792 | template <typename LHS_t, typename RHS_t, unsigned Opcode, |
| 793 | bool Commutable = false> |
| 794 | struct BinaryOp_match { |
| 795 | LHS_t L; |
| 796 | RHS_t R; |
| 797 | |
| 798 | // The evaluation order is always stable, regardless of Commutability. |
| 799 | // The LHS is always matched first. |
| 800 | BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
| 801 | |
| 802 | template <typename OpTy> bool match(OpTy *V) { |
| 803 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
| 804 | auto *I = cast<BinaryOperator>(V); |
| 805 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || |
| 806 | (Commutable && L.match(I->getOperand(1)) && |
| 807 | R.match(I->getOperand(0))); |
| 808 | } |
| 809 | if (auto *CE = dyn_cast<ConstantExpr>(V)) |
| 810 | return CE->getOpcode() == Opcode && |
| 811 | ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) || |
| 812 | (Commutable && L.match(CE->getOperand(1)) && |
| 813 | R.match(CE->getOperand(0)))); |
| 814 | return false; |
| 815 | } |
| 816 | }; |
| 817 | |
| 818 | template <typename LHS, typename RHS> |
| 819 | inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L, |
| 820 | const RHS &R) { |
| 821 | return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R); |
| 822 | } |
| 823 | |
| 824 | template <typename LHS, typename RHS> |
| 825 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L, |
| 826 | const RHS &R) { |
| 827 | return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R); |
| 828 | } |
| 829 | |
| 830 | template <typename LHS, typename RHS> |
| 831 | inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L, |
| 832 | const RHS &R) { |
| 833 | return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R); |
| 834 | } |
| 835 | |
| 836 | template <typename LHS, typename RHS> |
| 837 | inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L, |
| 838 | const RHS &R) { |
| 839 | return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R); |
| 840 | } |
| 841 | |
| 842 | template <typename Op_t> struct FNeg_match { |
| 843 | Op_t X; |
| 844 | |
| 845 | FNeg_match(const Op_t &Op) : X(Op) {} |
| 846 | template <typename OpTy> bool match(OpTy *V) { |
| 847 | auto *FPMO = dyn_cast<FPMathOperator>(V); |
| 848 | if (!FPMO) return false; |
| 849 | |
| 850 | if (FPMO->getOpcode() == Instruction::FNeg) |
| 851 | return X.match(FPMO->getOperand(0)); |
| 852 | |
| 853 | if (FPMO->getOpcode() == Instruction::FSub) { |
| 854 | if (FPMO->hasNoSignedZeros()) { |
| 855 | // With 'nsz', any zero goes. |
| 856 | if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0))) |
| 857 | return false; |
| 858 | } else { |
| 859 | // Without 'nsz', we need fsub -0.0, X exactly. |
| 860 | if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0))) |
| 861 | return false; |
| 862 | } |
| 863 | |
| 864 | return X.match(FPMO->getOperand(1)); |
| 865 | } |
| 866 | |
| 867 | return false; |
| 868 | } |
| 869 | }; |
| 870 | |
| 871 | /// Match 'fneg X' as 'fsub -0.0, X'. |
| 872 | template <typename OpTy> |
| 873 | inline FNeg_match<OpTy> |
| 874 | m_FNeg(const OpTy &X) { |
| 875 | return FNeg_match<OpTy>(X); |
| 876 | } |
| 877 | |
| 878 | /// Match 'fneg X' as 'fsub +-0.0, X'. |
| 879 | template <typename RHS> |
| 880 | inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub> |
| 881 | m_FNegNSZ(const RHS &X) { |
| 882 | return m_FSub(m_AnyZeroFP(), X); |
| 883 | } |
| 884 | |
| 885 | template <typename LHS, typename RHS> |
| 886 | inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L, |
| 887 | const RHS &R) { |
| 888 | return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R); |
| 889 | } |
| 890 | |
| 891 | template <typename LHS, typename RHS> |
| 892 | inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L, |
| 893 | const RHS &R) { |
| 894 | return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R); |
| 895 | } |
| 896 | |
| 897 | template <typename LHS, typename RHS> |
| 898 | inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L, |
| 899 | const RHS &R) { |
| 900 | return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R); |
| 901 | } |
| 902 | |
| 903 | template <typename LHS, typename RHS> |
| 904 | inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L, |
| 905 | const RHS &R) { |
| 906 | return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R); |
| 907 | } |
| 908 | |
| 909 | template <typename LHS, typename RHS> |
| 910 | inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L, |
| 911 | const RHS &R) { |
| 912 | return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R); |
| 913 | } |
| 914 | |
| 915 | template <typename LHS, typename RHS> |
| 916 | inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L, |
| 917 | const RHS &R) { |
| 918 | return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R); |
| 919 | } |
| 920 | |
| 921 | template <typename LHS, typename RHS> |
| 922 | inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L, |
| 923 | const RHS &R) { |
| 924 | return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R); |
| 925 | } |
| 926 | |
| 927 | template <typename LHS, typename RHS> |
| 928 | inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L, |
| 929 | const RHS &R) { |
| 930 | return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R); |
| 931 | } |
| 932 | |
| 933 | template <typename LHS, typename RHS> |
| 934 | inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L, |
| 935 | const RHS &R) { |
| 936 | return BinaryOp_match<LHS, RHS, Instruction::And>(L, R); |
| 937 | } |
| 938 | |
| 939 | template <typename LHS, typename RHS> |
| 940 | inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L, |
| 941 | const RHS &R) { |
| 942 | return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R); |
| 943 | } |
| 944 | |
| 945 | template <typename LHS, typename RHS> |
| 946 | inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L, |
| 947 | const RHS &R) { |
| 948 | return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R); |
| 949 | } |
| 950 | |
| 951 | template <typename LHS, typename RHS> |
| 952 | inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L, |
| 953 | const RHS &R) { |
| 954 | return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R); |
| 955 | } |
| 956 | |
| 957 | template <typename LHS, typename RHS> |
| 958 | inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L, |
| 959 | const RHS &R) { |
| 960 | return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R); |
| 961 | } |
| 962 | |
| 963 | template <typename LHS, typename RHS> |
| 964 | inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L, |
| 965 | const RHS &R) { |
| 966 | return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R); |
| 967 | } |
| 968 | |
| 969 | template <typename LHS_t, typename RHS_t, unsigned Opcode, |
| 970 | unsigned WrapFlags = 0> |
| 971 | struct OverflowingBinaryOp_match { |
| 972 | LHS_t L; |
| 973 | RHS_t R; |
| 974 | |
| 975 | OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) |
| 976 | : L(LHS), R(RHS) {} |
| 977 | |
| 978 | template <typename OpTy> bool match(OpTy *V) { |
| 979 | if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) { |
| 980 | if (Op->getOpcode() != Opcode) |
| 981 | return false; |
| 982 | if (WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap && |
| 983 | !Op->hasNoUnsignedWrap()) |
| 984 | return false; |
| 985 | if (WrapFlags & OverflowingBinaryOperator::NoSignedWrap && |
| 986 | !Op->hasNoSignedWrap()) |
| 987 | return false; |
| 988 | return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1)); |
| 989 | } |
| 990 | return false; |
| 991 | } |
| 992 | }; |
| 993 | |
| 994 | template <typename LHS, typename RHS> |
| 995 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
| 996 | OverflowingBinaryOperator::NoSignedWrap> |
| 997 | m_NSWAdd(const LHS &L, const RHS &R) { |
| 998 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
| 999 | OverflowingBinaryOperator::NoSignedWrap>( |
| 1000 | L, R); |
| 1001 | } |
| 1002 | template <typename LHS, typename RHS> |
| 1003 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
| 1004 | OverflowingBinaryOperator::NoSignedWrap> |
| 1005 | m_NSWSub(const LHS &L, const RHS &R) { |
| 1006 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
| 1007 | OverflowingBinaryOperator::NoSignedWrap>( |
| 1008 | L, R); |
| 1009 | } |
| 1010 | template <typename LHS, typename RHS> |
| 1011 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
| 1012 | OverflowingBinaryOperator::NoSignedWrap> |
| 1013 | m_NSWMul(const LHS &L, const RHS &R) { |
| 1014 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
| 1015 | OverflowingBinaryOperator::NoSignedWrap>( |
| 1016 | L, R); |
| 1017 | } |
| 1018 | template <typename LHS, typename RHS> |
| 1019 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
| 1020 | OverflowingBinaryOperator::NoSignedWrap> |
| 1021 | m_NSWShl(const LHS &L, const RHS &R) { |
| 1022 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
| 1023 | OverflowingBinaryOperator::NoSignedWrap>( |
| 1024 | L, R); |
| 1025 | } |
| 1026 | |
| 1027 | template <typename LHS, typename RHS> |
| 1028 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
| 1029 | OverflowingBinaryOperator::NoUnsignedWrap> |
| 1030 | m_NUWAdd(const LHS &L, const RHS &R) { |
| 1031 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, |
| 1032 | OverflowingBinaryOperator::NoUnsignedWrap>( |
| 1033 | L, R); |
| 1034 | } |
| 1035 | template <typename LHS, typename RHS> |
| 1036 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
| 1037 | OverflowingBinaryOperator::NoUnsignedWrap> |
| 1038 | m_NUWSub(const LHS &L, const RHS &R) { |
| 1039 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, |
| 1040 | OverflowingBinaryOperator::NoUnsignedWrap>( |
| 1041 | L, R); |
| 1042 | } |
| 1043 | template <typename LHS, typename RHS> |
| 1044 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
| 1045 | OverflowingBinaryOperator::NoUnsignedWrap> |
| 1046 | m_NUWMul(const LHS &L, const RHS &R) { |
| 1047 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, |
| 1048 | OverflowingBinaryOperator::NoUnsignedWrap>( |
| 1049 | L, R); |
| 1050 | } |
| 1051 | template <typename LHS, typename RHS> |
| 1052 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
| 1053 | OverflowingBinaryOperator::NoUnsignedWrap> |
| 1054 | m_NUWShl(const LHS &L, const RHS &R) { |
| 1055 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, |
| 1056 | OverflowingBinaryOperator::NoUnsignedWrap>( |
| 1057 | L, R); |
| 1058 | } |
| 1059 | |
| 1060 | //===----------------------------------------------------------------------===// |
| 1061 | // Class that matches a group of binary opcodes. |
| 1062 | // |
| 1063 | template <typename LHS_t, typename RHS_t, typename Predicate> |
| 1064 | struct BinOpPred_match : Predicate { |
| 1065 | LHS_t L; |
| 1066 | RHS_t R; |
| 1067 | |
| 1068 | BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
| 1069 | |
| 1070 | template <typename OpTy> bool match(OpTy *V) { |
| 1071 | if (auto *I = dyn_cast<Instruction>(V)) |
| 1072 | return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) && |
| 1073 | R.match(I->getOperand(1)); |
| 1074 | if (auto *CE = dyn_cast<ConstantExpr>(V)) |
| 1075 | return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) && |
| 1076 | R.match(CE->getOperand(1)); |
| 1077 | return false; |
| 1078 | } |
| 1079 | }; |
| 1080 | |
| 1081 | struct is_shift_op { |
| 1082 | bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); } |
| 1083 | }; |
| 1084 | |
| 1085 | struct is_right_shift_op { |
| 1086 | bool isOpType(unsigned Opcode) { |
| 1087 | return Opcode == Instruction::LShr || Opcode == Instruction::AShr; |
| 1088 | } |
| 1089 | }; |
| 1090 | |
| 1091 | struct is_logical_shift_op { |
| 1092 | bool isOpType(unsigned Opcode) { |
| 1093 | return Opcode == Instruction::LShr || Opcode == Instruction::Shl; |
| 1094 | } |
| 1095 | }; |
| 1096 | |
| 1097 | struct is_bitwiselogic_op { |
| 1098 | bool isOpType(unsigned Opcode) { |
| 1099 | return Instruction::isBitwiseLogicOp(Opcode); |
| 1100 | } |
| 1101 | }; |
| 1102 | |
| 1103 | struct is_idiv_op { |
| 1104 | bool isOpType(unsigned Opcode) { |
| 1105 | return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv; |
| 1106 | } |
| 1107 | }; |
| 1108 | |
| 1109 | struct is_irem_op { |
| 1110 | bool isOpType(unsigned Opcode) { |
| 1111 | return Opcode == Instruction::SRem || Opcode == Instruction::URem; |
| 1112 | } |
| 1113 | }; |
| 1114 | |
| 1115 | /// Matches shift operations. |
| 1116 | template <typename LHS, typename RHS> |
| 1117 | inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L, |
| 1118 | const RHS &R) { |
| 1119 | return BinOpPred_match<LHS, RHS, is_shift_op>(L, R); |
| 1120 | } |
| 1121 | |
| 1122 | /// Matches logical shift operations. |
| 1123 | template <typename LHS, typename RHS> |
| 1124 | inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L, |
| 1125 | const RHS &R) { |
| 1126 | return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R); |
| 1127 | } |
| 1128 | |
| 1129 | /// Matches logical shift operations. |
| 1130 | template <typename LHS, typename RHS> |
| 1131 | inline BinOpPred_match<LHS, RHS, is_logical_shift_op> |
| 1132 | m_LogicalShift(const LHS &L, const RHS &R) { |
| 1133 | return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R); |
| 1134 | } |
| 1135 | |
| 1136 | /// Matches bitwise logic operations. |
| 1137 | template <typename LHS, typename RHS> |
| 1138 | inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op> |
| 1139 | m_BitwiseLogic(const LHS &L, const RHS &R) { |
| 1140 | return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R); |
| 1141 | } |
| 1142 | |
| 1143 | /// Matches integer division operations. |
| 1144 | template <typename LHS, typename RHS> |
| 1145 | inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L, |
| 1146 | const RHS &R) { |
| 1147 | return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R); |
| 1148 | } |
| 1149 | |
| 1150 | /// Matches integer remainder operations. |
| 1151 | template <typename LHS, typename RHS> |
| 1152 | inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L, |
| 1153 | const RHS &R) { |
| 1154 | return BinOpPred_match<LHS, RHS, is_irem_op>(L, R); |
| 1155 | } |
| 1156 | |
| 1157 | //===----------------------------------------------------------------------===// |
| 1158 | // Class that matches exact binary ops. |
| 1159 | // |
| 1160 | template <typename SubPattern_t> struct Exact_match { |
| 1161 | SubPattern_t SubPattern; |
| 1162 | |
| 1163 | Exact_match(const SubPattern_t &SP) : SubPattern(SP) {} |
| 1164 | |
| 1165 | template <typename OpTy> bool match(OpTy *V) { |
| 1166 | if (auto *PEO = dyn_cast<PossiblyExactOperator>(V)) |
| 1167 | return PEO->isExact() && SubPattern.match(V); |
| 1168 | return false; |
| 1169 | } |
| 1170 | }; |
| 1171 | |
| 1172 | template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) { |
| 1173 | return SubPattern; |
| 1174 | } |
| 1175 | |
| 1176 | //===----------------------------------------------------------------------===// |
| 1177 | // Matchers for CmpInst classes |
| 1178 | // |
| 1179 | |
| 1180 | template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy, |
| 1181 | bool Commutable = false> |
| 1182 | struct CmpClass_match { |
| 1183 | PredicateTy &Predicate; |
| 1184 | LHS_t L; |
| 1185 | RHS_t R; |
| 1186 | |
| 1187 | // The evaluation order is always stable, regardless of Commutability. |
| 1188 | // The LHS is always matched first. |
| 1189 | CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS) |
| 1190 | : Predicate(Pred), L(LHS), R(RHS) {} |
| 1191 | |
| 1192 | template <typename OpTy> bool match(OpTy *V) { |
| 1193 | if (auto *I = dyn_cast<Class>(V)) { |
| 1194 | if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) { |
| 1195 | Predicate = I->getPredicate(); |
| 1196 | return true; |
| 1197 | } else if (Commutable && L.match(I->getOperand(1)) && |
| 1198 | R.match(I->getOperand(0))) { |
| 1199 | Predicate = I->getSwappedPredicate(); |
| 1200 | return true; |
| 1201 | } |
| 1202 | } |
| 1203 | return false; |
| 1204 | } |
| 1205 | }; |
| 1206 | |
| 1207 | template <typename LHS, typename RHS> |
| 1208 | inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate> |
| 1209 | m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
| 1210 | return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R); |
| 1211 | } |
| 1212 | |
| 1213 | template <typename LHS, typename RHS> |
| 1214 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate> |
| 1215 | m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
| 1216 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R); |
| 1217 | } |
| 1218 | |
| 1219 | template <typename LHS, typename RHS> |
| 1220 | inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate> |
| 1221 | m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
| 1222 | return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R); |
| 1223 | } |
| 1224 | |
| 1225 | //===----------------------------------------------------------------------===// |
| 1226 | // Matchers for instructions with a given opcode and number of operands. |
| 1227 | // |
| 1228 | |
| 1229 | /// Matches instructions with Opcode and three operands. |
| 1230 | template <typename T0, unsigned Opcode> struct OneOps_match { |
| 1231 | T0 Op1; |
| 1232 | |
| 1233 | OneOps_match(const T0 &Op1) : Op1(Op1) {} |
| 1234 | |
| 1235 | template <typename OpTy> bool match(OpTy *V) { |
| 1236 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
| 1237 | auto *I = cast<Instruction>(V); |
| 1238 | return Op1.match(I->getOperand(0)); |
| 1239 | } |
| 1240 | return false; |
| 1241 | } |
| 1242 | }; |
| 1243 | |
| 1244 | /// Matches instructions with Opcode and three operands. |
| 1245 | template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match { |
| 1246 | T0 Op1; |
| 1247 | T1 Op2; |
| 1248 | |
| 1249 | TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {} |
| 1250 | |
| 1251 | template <typename OpTy> bool match(OpTy *V) { |
| 1252 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
| 1253 | auto *I = cast<Instruction>(V); |
| 1254 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)); |
| 1255 | } |
| 1256 | return false; |
| 1257 | } |
| 1258 | }; |
| 1259 | |
| 1260 | /// Matches instructions with Opcode and three operands. |
| 1261 | template <typename T0, typename T1, typename T2, unsigned Opcode> |
| 1262 | struct ThreeOps_match { |
| 1263 | T0 Op1; |
| 1264 | T1 Op2; |
| 1265 | T2 Op3; |
| 1266 | |
| 1267 | ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3) |
| 1268 | : Op1(Op1), Op2(Op2), Op3(Op3) {} |
| 1269 | |
| 1270 | template <typename OpTy> bool match(OpTy *V) { |
| 1271 | if (V->getValueID() == Value::InstructionVal + Opcode) { |
| 1272 | auto *I = cast<Instruction>(V); |
| 1273 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && |
| 1274 | Op3.match(I->getOperand(2)); |
| 1275 | } |
| 1276 | return false; |
| 1277 | } |
| 1278 | }; |
| 1279 | |
| 1280 | /// Matches SelectInst. |
| 1281 | template <typename Cond, typename LHS, typename RHS> |
| 1282 | inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select> |
| 1283 | m_Select(const Cond &C, const LHS &L, const RHS &R) { |
| 1284 | return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R); |
| 1285 | } |
| 1286 | |
| 1287 | /// This matches a select of two constants, e.g.: |
| 1288 | /// m_SelectCst<-1, 0>(m_Value(V)) |
| 1289 | template <int64_t L, int64_t R, typename Cond> |
| 1290 | inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>, |
| 1291 | Instruction::Select> |
| 1292 | m_SelectCst(const Cond &C) { |
| 1293 | return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>()); |
| 1294 | } |
| 1295 | |
| 1296 | /// Matches FreezeInst. |
| 1297 | template <typename OpTy> |
| 1298 | inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) { |
| 1299 | return OneOps_match<OpTy, Instruction::Freeze>(Op); |
| 1300 | } |
| 1301 | |
| 1302 | /// Matches InsertElementInst. |
| 1303 | template <typename Val_t, typename Elt_t, typename Idx_t> |
| 1304 | inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement> |
| 1305 | m_InsertElement(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) { |
| 1306 | return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>( |
| 1307 | Val, Elt, Idx); |
| 1308 | } |
| 1309 | |
| 1310 | /// Matches ExtractElementInst. |
| 1311 | template <typename Val_t, typename Idx_t> |
| 1312 | inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement> |
| 1313 | m_ExtractElement(const Val_t &Val, const Idx_t &Idx) { |
| 1314 | return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx); |
| 1315 | } |
| 1316 | |
| 1317 | /// Matches ShuffleVectorInst. |
| 1318 | template <typename V1_t, typename V2_t, typename Mask_t> |
| 1319 | inline ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector> |
| 1320 | m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) { |
| 1321 | return ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>(v1, v2, |
| 1322 | m); |
| 1323 | } |
| 1324 | |
| 1325 | /// Matches LoadInst. |
| 1326 | template <typename OpTy> |
| 1327 | inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) { |
| 1328 | return OneOps_match<OpTy, Instruction::Load>(Op); |
| 1329 | } |
| 1330 | |
| 1331 | /// Matches StoreInst. |
| 1332 | template <typename ValueOpTy, typename PointerOpTy> |
| 1333 | inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store> |
| 1334 | m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) { |
| 1335 | return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp, |
| 1336 | PointerOp); |
| 1337 | } |
| 1338 | |
| 1339 | //===----------------------------------------------------------------------===// |
| 1340 | // Matchers for CastInst classes |
| 1341 | // |
| 1342 | |
| 1343 | template <typename Op_t, unsigned Opcode> struct CastClass_match { |
| 1344 | Op_t Op; |
| 1345 | |
| 1346 | CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {} |
| 1347 | |
| 1348 | template <typename OpTy> bool match(OpTy *V) { |
| 1349 | if (auto *O = dyn_cast<Operator>(V)) |
| 1350 | return O->getOpcode() == Opcode && Op.match(O->getOperand(0)); |
| 1351 | return false; |
| 1352 | } |
| 1353 | }; |
| 1354 | |
| 1355 | /// Matches BitCast. |
| 1356 | template <typename OpTy> |
| 1357 | inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) { |
| 1358 | return CastClass_match<OpTy, Instruction::BitCast>(Op); |
| 1359 | } |
| 1360 | |
| 1361 | /// Matches PtrToInt. |
| 1362 | template <typename OpTy> |
| 1363 | inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) { |
| 1364 | return CastClass_match<OpTy, Instruction::PtrToInt>(Op); |
| 1365 | } |
| 1366 | |
| 1367 | /// Matches Trunc. |
| 1368 | template <typename OpTy> |
| 1369 | inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) { |
| 1370 | return CastClass_match<OpTy, Instruction::Trunc>(Op); |
| 1371 | } |
| 1372 | |
| 1373 | template <typename OpTy> |
| 1374 | inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy> |
| 1375 | m_TruncOrSelf(const OpTy &Op) { |
| 1376 | return m_CombineOr(m_Trunc(Op), Op); |
| 1377 | } |
| 1378 | |
| 1379 | /// Matches SExt. |
| 1380 | template <typename OpTy> |
| 1381 | inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) { |
| 1382 | return CastClass_match<OpTy, Instruction::SExt>(Op); |
| 1383 | } |
| 1384 | |
| 1385 | /// Matches ZExt. |
| 1386 | template <typename OpTy> |
| 1387 | inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { |
| 1388 | return CastClass_match<OpTy, Instruction::ZExt>(Op); |
| 1389 | } |
| 1390 | |
| 1391 | template <typename OpTy> |
| 1392 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy> |
| 1393 | m_ZExtOrSelf(const OpTy &Op) { |
| 1394 | return m_CombineOr(m_ZExt(Op), Op); |
| 1395 | } |
| 1396 | |
| 1397 | template <typename OpTy> |
| 1398 | inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy> |
| 1399 | m_SExtOrSelf(const OpTy &Op) { |
| 1400 | return m_CombineOr(m_SExt(Op), Op); |
| 1401 | } |
| 1402 | |
| 1403 | template <typename OpTy> |
| 1404 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, |
| 1405 | CastClass_match<OpTy, Instruction::SExt>> |
| 1406 | m_ZExtOrSExt(const OpTy &Op) { |
| 1407 | return m_CombineOr(m_ZExt(Op), m_SExt(Op)); |
| 1408 | } |
| 1409 | |
| 1410 | template <typename OpTy> |
| 1411 | inline match_combine_or< |
| 1412 | match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, |
| 1413 | CastClass_match<OpTy, Instruction::SExt>>, |
| 1414 | OpTy> |
| 1415 | m_ZExtOrSExtOrSelf(const OpTy &Op) { |
| 1416 | return m_CombineOr(m_ZExtOrSExt(Op), Op); |
| 1417 | } |
| 1418 | |
| 1419 | /// Matches UIToFP. |
| 1420 | template <typename OpTy> |
| 1421 | inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) { |
| 1422 | return CastClass_match<OpTy, Instruction::UIToFP>(Op); |
| 1423 | } |
| 1424 | |
| 1425 | /// Matches SIToFP. |
| 1426 | template <typename OpTy> |
| 1427 | inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) { |
| 1428 | return CastClass_match<OpTy, Instruction::SIToFP>(Op); |
| 1429 | } |
| 1430 | |
| 1431 | /// Matches FPTrunc |
| 1432 | template <typename OpTy> |
| 1433 | inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) { |
| 1434 | return CastClass_match<OpTy, Instruction::FPTrunc>(Op); |
| 1435 | } |
| 1436 | |
| 1437 | /// Matches FPExt |
| 1438 | template <typename OpTy> |
| 1439 | inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) { |
| 1440 | return CastClass_match<OpTy, Instruction::FPExt>(Op); |
| 1441 | } |
| 1442 | |
| 1443 | //===----------------------------------------------------------------------===// |
| 1444 | // Matchers for control flow. |
| 1445 | // |
| 1446 | |
| 1447 | struct br_match { |
| 1448 | BasicBlock *&Succ; |
| 1449 | |
| 1450 | br_match(BasicBlock *&Succ) : Succ(Succ) {} |
| 1451 | |
| 1452 | template <typename OpTy> bool match(OpTy *V) { |
| 1453 | if (auto *BI = dyn_cast<BranchInst>(V)) |
| 1454 | if (BI->isUnconditional()) { |
| 1455 | Succ = BI->getSuccessor(0); |
| 1456 | return true; |
| 1457 | } |
| 1458 | return false; |
| 1459 | } |
| 1460 | }; |
| 1461 | |
| 1462 | inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } |
| 1463 | |
| 1464 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> |
| 1465 | struct brc_match { |
| 1466 | Cond_t Cond; |
| 1467 | TrueBlock_t T; |
| 1468 | FalseBlock_t F; |
| 1469 | |
| 1470 | brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f) |
| 1471 | : Cond(C), T(t), F(f) {} |
| 1472 | |
| 1473 | template <typename OpTy> bool match(OpTy *V) { |
| 1474 | if (auto *BI = dyn_cast<BranchInst>(V)) |
| 1475 | if (BI->isConditional() && Cond.match(BI->getCondition())) |
| 1476 | return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1)); |
| 1477 | return false; |
| 1478 | } |
| 1479 | }; |
| 1480 | |
| 1481 | template <typename Cond_t> |
| 1482 | inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>> |
| 1483 | m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { |
| 1484 | return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>( |
| 1485 | C, m_BasicBlock(T), m_BasicBlock(F)); |
| 1486 | } |
| 1487 | |
| 1488 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> |
| 1489 | inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t> |
| 1490 | m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) { |
| 1491 | return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F); |
| 1492 | } |
| 1493 | |
| 1494 | //===----------------------------------------------------------------------===// |
| 1495 | // Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y). |
| 1496 | // |
| 1497 | |
| 1498 | template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t, |
| 1499 | bool Commutable = false> |
| 1500 | struct MaxMin_match { |
| 1501 | LHS_t L; |
| 1502 | RHS_t R; |
| 1503 | |
| 1504 | // The evaluation order is always stable, regardless of Commutability. |
| 1505 | // The LHS is always matched first. |
| 1506 | MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} |
| 1507 | |
| 1508 | template <typename OpTy> bool match(OpTy *V) { |
| 1509 | // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x". |
| 1510 | auto *SI = dyn_cast<SelectInst>(V); |
| 1511 | if (!SI) |
| 1512 | return false; |
| 1513 | auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition()); |
| 1514 | if (!Cmp) |
| 1515 | return false; |
| 1516 | // At this point we have a select conditioned on a comparison. Check that |
| 1517 | // it is the values returned by the select that are being compared. |
| 1518 | Value *TrueVal = SI->getTrueValue(); |
| 1519 | Value *FalseVal = SI->getFalseValue(); |
| 1520 | Value *LHS = Cmp->getOperand(0); |
| 1521 | Value *RHS = Cmp->getOperand(1); |
| 1522 | if ((TrueVal != LHS || FalseVal != RHS) && |
| 1523 | (TrueVal != RHS || FalseVal != LHS)) |
| 1524 | return false; |
| 1525 | typename CmpInst_t::Predicate Pred = |
| 1526 | LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate(); |
| 1527 | // Does "(x pred y) ? x : y" represent the desired max/min operation? |
| 1528 | if (!Pred_t::match(Pred)) |
| 1529 | return false; |
| 1530 | // It does! Bind the operands. |
| 1531 | return (L.match(LHS) && R.match(RHS)) || |
| 1532 | (Commutable && L.match(RHS) && R.match(LHS)); |
| 1533 | } |
| 1534 | }; |
| 1535 | |
| 1536 | /// Helper class for identifying signed max predicates. |
| 1537 | struct smax_pred_ty { |
| 1538 | static bool match(ICmpInst::Predicate Pred) { |
| 1539 | return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE; |
| 1540 | } |
| 1541 | }; |
| 1542 | |
| 1543 | /// Helper class for identifying signed min predicates. |
| 1544 | struct smin_pred_ty { |
| 1545 | static bool match(ICmpInst::Predicate Pred) { |
| 1546 | return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE; |
| 1547 | } |
| 1548 | }; |
| 1549 | |
| 1550 | /// Helper class for identifying unsigned max predicates. |
| 1551 | struct umax_pred_ty { |
| 1552 | static bool match(ICmpInst::Predicate Pred) { |
| 1553 | return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE; |
| 1554 | } |
| 1555 | }; |
| 1556 | |
| 1557 | /// Helper class for identifying unsigned min predicates. |
| 1558 | struct umin_pred_ty { |
| 1559 | static bool match(ICmpInst::Predicate Pred) { |
| 1560 | return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE; |
| 1561 | } |
| 1562 | }; |
| 1563 | |
| 1564 | /// Helper class for identifying ordered max predicates. |
| 1565 | struct ofmax_pred_ty { |
| 1566 | static bool match(FCmpInst::Predicate Pred) { |
| 1567 | return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE; |
| 1568 | } |
| 1569 | }; |
| 1570 | |
| 1571 | /// Helper class for identifying ordered min predicates. |
| 1572 | struct ofmin_pred_ty { |
| 1573 | static bool match(FCmpInst::Predicate Pred) { |
| 1574 | return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE; |
| 1575 | } |
| 1576 | }; |
| 1577 | |
| 1578 | /// Helper class for identifying unordered max predicates. |
| 1579 | struct ufmax_pred_ty { |
| 1580 | static bool match(FCmpInst::Predicate Pred) { |
| 1581 | return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE; |
| 1582 | } |
| 1583 | }; |
| 1584 | |
| 1585 | /// Helper class for identifying unordered min predicates. |
| 1586 | struct ufmin_pred_ty { |
| 1587 | static bool match(FCmpInst::Predicate Pred) { |
| 1588 | return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE; |
| 1589 | } |
| 1590 | }; |
| 1591 | |
| 1592 | template <typename LHS, typename RHS> |
| 1593 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L, |
| 1594 | const RHS &R) { |
| 1595 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R); |
| 1596 | } |
| 1597 | |
| 1598 | template <typename LHS, typename RHS> |
| 1599 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L, |
| 1600 | const RHS &R) { |
| 1601 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R); |
| 1602 | } |
| 1603 | |
| 1604 | template <typename LHS, typename RHS> |
| 1605 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L, |
| 1606 | const RHS &R) { |
| 1607 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R); |
| 1608 | } |
| 1609 | |
| 1610 | template <typename LHS, typename RHS> |
| 1611 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L, |
| 1612 | const RHS &R) { |
| 1613 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R); |
| 1614 | } |
| 1615 | |
| 1616 | /// Match an 'ordered' floating point maximum function. |
| 1617 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
| 1618 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
| 1619 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' |
| 1620 | /// semantics. In the presence of 'NaN' we have to preserve the original |
| 1621 | /// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate. |
| 1622 | /// |
| 1623 | /// max(L, R) iff L and R are not NaN |
| 1624 | /// m_OrdFMax(L, R) = R iff L or R are NaN |
| 1625 | template <typename LHS, typename RHS> |
| 1626 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L, |
| 1627 | const RHS &R) { |
| 1628 | return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R); |
| 1629 | } |
| 1630 | |
| 1631 | /// Match an 'ordered' floating point minimum function. |
| 1632 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
| 1633 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
| 1634 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' |
| 1635 | /// semantics. In the presence of 'NaN' we have to preserve the original |
| 1636 | /// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate. |
| 1637 | /// |
| 1638 | /// min(L, R) iff L and R are not NaN |
| 1639 | /// m_OrdFMin(L, R) = R iff L or R are NaN |
| 1640 | template <typename LHS, typename RHS> |
| 1641 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L, |
| 1642 | const RHS &R) { |
| 1643 | return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R); |
| 1644 | } |
| 1645 | |
| 1646 | /// Match an 'unordered' floating point maximum function. |
| 1647 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
| 1648 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
| 1649 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' |
| 1650 | /// semantics. In the presence of 'NaN' we have to preserve the original |
| 1651 | /// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate. |
| 1652 | /// |
| 1653 | /// max(L, R) iff L and R are not NaN |
| 1654 | /// m_UnordFMax(L, R) = L iff L or R are NaN |
| 1655 | template <typename LHS, typename RHS> |
| 1656 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty> |
| 1657 | m_UnordFMax(const LHS &L, const RHS &R) { |
| 1658 | return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R); |
| 1659 | } |
| 1660 | |
| 1661 | /// Match an 'unordered' floating point minimum function. |
| 1662 | /// Floating point has one special value 'NaN'. Therefore, there is no total |
| 1663 | /// order. However, if we can ignore the 'NaN' value (for example, because of a |
| 1664 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' |
| 1665 | /// semantics. In the presence of 'NaN' we have to preserve the original |
| 1666 | /// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate. |
| 1667 | /// |
| 1668 | /// min(L, R) iff L and R are not NaN |
| 1669 | /// m_UnordFMin(L, R) = L iff L or R are NaN |
| 1670 | template <typename LHS, typename RHS> |
| 1671 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty> |
| 1672 | m_UnordFMin(const LHS &L, const RHS &R) { |
| 1673 | return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R); |
| 1674 | } |
| 1675 | |
| 1676 | //===----------------------------------------------------------------------===// |
| 1677 | // Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b |
| 1678 | // Note that S might be matched to other instructions than AddInst. |
| 1679 | // |
| 1680 | |
| 1681 | template <typename LHS_t, typename RHS_t, typename Sum_t> |
| 1682 | struct UAddWithOverflow_match { |
| 1683 | LHS_t L; |
| 1684 | RHS_t R; |
| 1685 | Sum_t S; |
| 1686 | |
| 1687 | UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S) |
| 1688 | : L(L), R(R), S(S) {} |
| 1689 | |
| 1690 | template <typename OpTy> bool match(OpTy *V) { |
| 1691 | Value *ICmpLHS, *ICmpRHS; |
| 1692 | ICmpInst::Predicate Pred; |
| 1693 | if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V)) |
| 1694 | return false; |
| 1695 | |
| 1696 | Value *AddLHS, *AddRHS; |
| 1697 | auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS)); |
| 1698 | |
| 1699 | // (a + b) u< a, (a + b) u< b |
| 1700 | if (Pred == ICmpInst::ICMP_ULT) |
| 1701 | if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS)) |
| 1702 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); |
| 1703 | |
| 1704 | // a >u (a + b), b >u (a + b) |
| 1705 | if (Pred == ICmpInst::ICMP_UGT) |
| 1706 | if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS)) |
| 1707 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); |
| 1708 | |
| 1709 | Value *Op1; |
| 1710 | auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes())); |
| 1711 | // (a ^ -1) <u b |
| 1712 | if (Pred == ICmpInst::ICMP_ULT) { |
| 1713 | if (XorExpr.match(ICmpLHS)) |
| 1714 | return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS); |
| 1715 | } |
| 1716 | // b > u (a ^ -1) |
| 1717 | if (Pred == ICmpInst::ICMP_UGT) { |
| 1718 | if (XorExpr.match(ICmpRHS)) |
| 1719 | return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS); |
| 1720 | } |
| 1721 | |
| 1722 | // Match special-case for increment-by-1. |
| 1723 | if (Pred == ICmpInst::ICMP_EQ) { |
| 1724 | // (a + 1) == 0 |
| 1725 | // (1 + a) == 0 |
| 1726 | if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) && |
| 1727 | (m_One().match(AddLHS) || m_One().match(AddRHS))) |
| 1728 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); |
| 1729 | // 0 == (a + 1) |
| 1730 | // 0 == (1 + a) |
| 1731 | if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) && |
| 1732 | (m_One().match(AddLHS) || m_One().match(AddRHS))) |
| 1733 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); |
| 1734 | } |
| 1735 | |
| 1736 | return false; |
| 1737 | } |
| 1738 | }; |
| 1739 | |
| 1740 | /// Match an icmp instruction checking for unsigned overflow on addition. |
| 1741 | /// |
| 1742 | /// S is matched to the addition whose result is being checked for overflow, and |
| 1743 | /// L and R are matched to the LHS and RHS of S. |
| 1744 | template <typename LHS_t, typename RHS_t, typename Sum_t> |
| 1745 | UAddWithOverflow_match<LHS_t, RHS_t, Sum_t> |
| 1746 | m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) { |
| 1747 | return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S); |
| 1748 | } |
| 1749 | |
| 1750 | template <typename Opnd_t> struct Argument_match { |
| 1751 | unsigned OpI; |
| 1752 | Opnd_t Val; |
| 1753 | |
| 1754 | Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} |
| 1755 | |
| 1756 | template <typename OpTy> bool match(OpTy *V) { |
| 1757 | // FIXME: Should likely be switched to use `CallBase`. |
| 1758 | if (const auto *CI = dyn_cast<CallInst>(V)) |
| 1759 | return Val.match(CI->getArgOperand(OpI)); |
| 1760 | return false; |
| 1761 | } |
| 1762 | }; |
| 1763 | |
| 1764 | /// Match an argument. |
| 1765 | template <unsigned OpI, typename Opnd_t> |
| 1766 | inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) { |
| 1767 | return Argument_match<Opnd_t>(OpI, Op); |
| 1768 | } |
| 1769 | |
| 1770 | /// Intrinsic matchers. |
| 1771 | struct IntrinsicID_match { |
| 1772 | unsigned ID; |
| 1773 | |
| 1774 | IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} |
| 1775 | |
| 1776 | template <typename OpTy> bool match(OpTy *V) { |
| 1777 | if (const auto *CI = dyn_cast<CallInst>(V)) |
| 1778 | if (const auto *F = CI->getCalledFunction()) |
| 1779 | return F->getIntrinsicID() == ID; |
| 1780 | return false; |
| 1781 | } |
| 1782 | }; |
| 1783 | |
| 1784 | /// Intrinsic matches are combinations of ID matchers, and argument |
| 1785 | /// matchers. Higher arity matcher are defined recursively in terms of and-ing |
| 1786 | /// them with lower arity matchers. Here's some convenient typedefs for up to |
| 1787 | /// several arguments, and more can be added as needed |
| 1788 | template <typename T0 = void, typename T1 = void, typename T2 = void, |
| 1789 | typename T3 = void, typename T4 = void, typename T5 = void, |
| 1790 | typename T6 = void, typename T7 = void, typename T8 = void, |
| 1791 | typename T9 = void, typename T10 = void> |
| 1792 | struct m_Intrinsic_Ty; |
| 1793 | template <typename T0> struct m_Intrinsic_Ty<T0> { |
| 1794 | using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>; |
| 1795 | }; |
| 1796 | template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { |
| 1797 | using Ty = |
| 1798 | match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>; |
| 1799 | }; |
| 1800 | template <typename T0, typename T1, typename T2> |
| 1801 | struct m_Intrinsic_Ty<T0, T1, T2> { |
| 1802 | using Ty = |
| 1803 | match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, |
| 1804 | Argument_match<T2>>; |
| 1805 | }; |
| 1806 | template <typename T0, typename T1, typename T2, typename T3> |
| 1807 | struct m_Intrinsic_Ty<T0, T1, T2, T3> { |
| 1808 | using Ty = |
| 1809 | match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, |
| 1810 | Argument_match<T3>>; |
| 1811 | }; |
| 1812 | |
| 1813 | template <typename T0, typename T1, typename T2, typename T3, typename T4> |
| 1814 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { |
| 1815 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty, |
| 1816 | Argument_match<T4>>; |
| 1817 | }; |
| 1818 | |
| 1819 | /// Match intrinsic calls like this: |
| 1820 | /// m_Intrinsic<Intrinsic::fabs>(m_Value(X)) |
| 1821 | template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { |
| 1822 | return IntrinsicID_match(IntrID); |
| 1823 | } |
| 1824 | |
| 1825 | template <Intrinsic::ID IntrID, typename T0> |
| 1826 | inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { |
| 1827 | return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); |
| 1828 | } |
| 1829 | |
| 1830 | template <Intrinsic::ID IntrID, typename T0, typename T1> |
| 1831 | inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0, |
| 1832 | const T1 &Op1) { |
| 1833 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1)); |
| 1834 | } |
| 1835 | |
| 1836 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2> |
| 1837 | inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty |
| 1838 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { |
| 1839 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2)); |
| 1840 | } |
| 1841 | |
| 1842 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
| 1843 | typename T3> |
| 1844 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty |
| 1845 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { |
| 1846 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); |
| 1847 | } |
| 1848 | |
| 1849 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, |
| 1850 | typename T3, typename T4> |
| 1851 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty |
| 1852 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, |
| 1853 | const T4 &Op4) { |
| 1854 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3), |
| 1855 | m_Argument<4>(Op4)); |
| 1856 | } |
| 1857 | |
| 1858 | // Helper intrinsic matching specializations. |
| 1859 | template <typename Opnd0> |
| 1860 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) { |
| 1861 | return m_Intrinsic<Intrinsic::bitreverse>(Op0); |
| 1862 | } |
| 1863 | |
| 1864 | template <typename Opnd0> |
| 1865 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) { |
| 1866 | return m_Intrinsic<Intrinsic::bswap>(Op0); |
| 1867 | } |
| 1868 | |
| 1869 | template <typename Opnd0> |
| 1870 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) { |
| 1871 | return m_Intrinsic<Intrinsic::fabs>(Op0); |
| 1872 | } |
| 1873 | |
| 1874 | template <typename Opnd0> |
| 1875 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) { |
| 1876 | return m_Intrinsic<Intrinsic::canonicalize>(Op0); |
| 1877 | } |
| 1878 | |
| 1879 | template <typename Opnd0, typename Opnd1> |
| 1880 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0, |
| 1881 | const Opnd1 &Op1) { |
| 1882 | return m_Intrinsic<Intrinsic::minnum>(Op0, Op1); |
| 1883 | } |
| 1884 | |
| 1885 | template <typename Opnd0, typename Opnd1> |
| 1886 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0, |
| 1887 | const Opnd1 &Op1) { |
| 1888 | return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1); |
| 1889 | } |
| 1890 | |
| 1891 | //===----------------------------------------------------------------------===// |
| 1892 | // Matchers for two-operands operators with the operators in either order |
| 1893 | // |
| 1894 | |
| 1895 | /// Matches a BinaryOperator with LHS and RHS in either order. |
| 1896 | template <typename LHS, typename RHS> |
| 1897 | inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) { |
| 1898 | return AnyBinaryOp_match<LHS, RHS, true>(L, R); |
| 1899 | } |
| 1900 | |
| 1901 | /// Matches an ICmp with a predicate over LHS and RHS in either order. |
| 1902 | /// Swaps the predicate if operands are commuted. |
| 1903 | template <typename LHS, typename RHS> |
| 1904 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true> |
| 1905 | m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { |
| 1906 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L, |
| 1907 | R); |
| 1908 | } |
| 1909 | |
| 1910 | /// Matches a Add with LHS and RHS in either order. |
| 1911 | template <typename LHS, typename RHS> |
| 1912 | inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L, |
| 1913 | const RHS &R) { |
| 1914 | return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R); |
| 1915 | } |
| 1916 | |
| 1917 | /// Matches a Mul with LHS and RHS in either order. |
| 1918 | template <typename LHS, typename RHS> |
| 1919 | inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L, |
| 1920 | const RHS &R) { |
| 1921 | return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R); |
| 1922 | } |
| 1923 | |
| 1924 | /// Matches an And with LHS and RHS in either order. |
| 1925 | template <typename LHS, typename RHS> |
| 1926 | inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L, |
| 1927 | const RHS &R) { |
| 1928 | return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R); |
| 1929 | } |
| 1930 | |
| 1931 | /// Matches an Or with LHS and RHS in either order. |
| 1932 | template <typename LHS, typename RHS> |
| 1933 | inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L, |
| 1934 | const RHS &R) { |
| 1935 | return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R); |
| 1936 | } |
| 1937 | |
| 1938 | /// Matches an Xor with LHS and RHS in either order. |
| 1939 | template <typename LHS, typename RHS> |
| 1940 | inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L, |
| 1941 | const RHS &R) { |
| 1942 | return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R); |
| 1943 | } |
| 1944 | |
| 1945 | /// Matches a 'Neg' as 'sub 0, V'. |
| 1946 | template <typename ValTy> |
| 1947 | inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub> |
| 1948 | m_Neg(const ValTy &V) { |
| 1949 | return m_Sub(m_ZeroInt(), V); |
| 1950 | } |
| 1951 | |
| 1952 | /// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'. |
| 1953 | template <typename ValTy> |
| 1954 | inline BinaryOp_match<ValTy, cst_pred_ty<is_all_ones>, Instruction::Xor, true> |
| 1955 | m_Not(const ValTy &V) { |
| 1956 | return m_c_Xor(V, m_AllOnes()); |
| 1957 | } |
| 1958 | |
| 1959 | /// Matches an SMin with LHS and RHS in either order. |
| 1960 | template <typename LHS, typename RHS> |
| 1961 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true> |
| 1962 | m_c_SMin(const LHS &L, const RHS &R) { |
| 1963 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R); |
| 1964 | } |
| 1965 | /// Matches an SMax with LHS and RHS in either order. |
| 1966 | template <typename LHS, typename RHS> |
| 1967 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true> |
| 1968 | m_c_SMax(const LHS &L, const RHS &R) { |
| 1969 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R); |
| 1970 | } |
| 1971 | /// Matches a UMin with LHS and RHS in either order. |
| 1972 | template <typename LHS, typename RHS> |
| 1973 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true> |
| 1974 | m_c_UMin(const LHS &L, const RHS &R) { |
| 1975 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R); |
| 1976 | } |
| 1977 | /// Matches a UMax with LHS and RHS in either order. |
| 1978 | template <typename LHS, typename RHS> |
| 1979 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true> |
| 1980 | m_c_UMax(const LHS &L, const RHS &R) { |
| 1981 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R); |
| 1982 | } |
| 1983 | |
| 1984 | /// Matches FAdd with LHS and RHS in either order. |
| 1985 | template <typename LHS, typename RHS> |
| 1986 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true> |
| 1987 | m_c_FAdd(const LHS &L, const RHS &R) { |
| 1988 | return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R); |
| 1989 | } |
| 1990 | |
| 1991 | /// Matches FMul with LHS and RHS in either order. |
| 1992 | template <typename LHS, typename RHS> |
| 1993 | inline BinaryOp_match<LHS, RHS, Instruction::FMul, true> |
| 1994 | m_c_FMul(const LHS &L, const RHS &R) { |
| 1995 | return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R); |
| 1996 | } |
| 1997 | |
| 1998 | template <typename Opnd_t> struct Signum_match { |
| 1999 | Opnd_t Val; |
| 2000 | Signum_match(const Opnd_t &V) : Val(V) {} |
| 2001 | |
| 2002 | template <typename OpTy> bool match(OpTy *V) { |
| 2003 | unsigned TypeSize = V->getType()->getScalarSizeInBits(); |
| 2004 | if (TypeSize == 0) |
| 2005 | return false; |
| 2006 | |
| 2007 | unsigned ShiftWidth = TypeSize - 1; |
| 2008 | Value *OpL = nullptr, *OpR = nullptr; |
| 2009 | |
| 2010 | // This is the representation of signum we match: |
| 2011 | // |
| 2012 | // signum(x) == (x >> 63) | (-x >>u 63) |
| 2013 | // |
| 2014 | // An i1 value is its own signum, so it's correct to match |
| 2015 | // |
| 2016 | // signum(x) == (x >> 0) | (-x >>u 0) |
| 2017 | // |
| 2018 | // for i1 values. |
| 2019 | |
| 2020 | auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth)); |
| 2021 | auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth)); |
| 2022 | auto Signum = m_Or(LHS, RHS); |
| 2023 | |
| 2024 | return Signum.match(V) && OpL == OpR && Val.match(OpL); |
| 2025 | } |
| 2026 | }; |
| 2027 | |
| 2028 | /// Matches a signum pattern. |
| 2029 | /// |
| 2030 | /// signum(x) = |
| 2031 | /// x > 0 -> 1 |
| 2032 | /// x == 0 -> 0 |
| 2033 | /// x < 0 -> -1 |
| 2034 | template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { |
| 2035 | return Signum_match<Val_t>(V); |
| 2036 | } |
| 2037 | |
| 2038 | template <int Ind, typename Opnd_t> struct ExtractValue_match { |
| 2039 | Opnd_t Val; |
| 2040 | ExtractValue_match(const Opnd_t &V) : Val(V) {} |
| 2041 | |
| 2042 | template <typename OpTy> bool match(OpTy *V) { |
| 2043 | if (auto *I = dyn_cast<ExtractValueInst>(V)) |
| 2044 | return I->getNumIndices() == 1 && I->getIndices()[0] == Ind && |
| 2045 | Val.match(I->getAggregateOperand()); |
| 2046 | return false; |
| 2047 | } |
| 2048 | }; |
| 2049 | |
| 2050 | /// Match a single index ExtractValue instruction. |
| 2051 | /// For example m_ExtractValue<1>(...) |
| 2052 | template <int Ind, typename Val_t> |
| 2053 | inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) { |
| 2054 | return ExtractValue_match<Ind, Val_t>(V); |
| 2055 | } |
| 2056 | |
| 2057 | /// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or |
| 2058 | /// the constant expression |
| 2059 | /// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>` |
| 2060 | /// under the right conditions determined by DataLayout. |
| 2061 | struct VScaleVal_match { |
| 2062 | private: |
| 2063 | template <typename Base, typename Offset> |
| 2064 | inline BinaryOp_match<Base, Offset, Instruction::GetElementPtr> |
| 2065 | m_OffsetGep(const Base &B, const Offset &O) { |
| 2066 | return BinaryOp_match<Base, Offset, Instruction::GetElementPtr>(B, O); |
| 2067 | } |
| 2068 | |
| 2069 | public: |
| 2070 | const DataLayout &DL; |
| 2071 | VScaleVal_match(const DataLayout &DL) : DL(DL) {} |
| 2072 | |
| 2073 | template <typename ITy> bool match(ITy *V) { |
| 2074 | if (m_Intrinsic<Intrinsic::vscale>().match(V)) |
| 2075 | return true; |
| 2076 | |
| 2077 | if (m_PtrToInt(m_OffsetGep(m_Zero(), m_SpecificInt(1))).match(V)) { |
| 2078 | Type *PtrTy = cast<Operator>(V)->getOperand(0)->getType(); |
| 2079 | Type *DerefTy = PtrTy->getPointerElementType(); |
| 2080 | if (DerefTy->isVectorTy() && DerefTy->getVectorIsScalable() && |
| 2081 | DL.getTypeAllocSizeInBits(DerefTy).getKnownMinSize() == 8) |
| 2082 | return true; |
| 2083 | } |
| 2084 | |
| 2085 | return false; |
| 2086 | } |
| 2087 | }; |
| 2088 | |
| 2089 | inline VScaleVal_match m_VScale(const DataLayout &DL) { |
| 2090 | return VScaleVal_match(DL); |
| 2091 | } |
| 2092 | |
| 2093 | } // end namespace PatternMatch |
| 2094 | } // end namespace llvm |
| 2095 | |
| 2096 | #endif // LLVM_IR_PATTERNMATCH_H |