| File: | build/source/llvm/include/llvm/IR/PatternMatch.h |
| Warning: | line 240, column 9 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- InstCombineAndOrXor.cpp --------------------------------------------===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // This file implements the visitAnd, visitOr, and visitXor functions. | ||||||||
| 10 | // | ||||||||
| 11 | //===----------------------------------------------------------------------===// | ||||||||
| 12 | |||||||||
| 13 | #include "InstCombineInternal.h" | ||||||||
| 14 | #include "llvm/Analysis/CmpInstAnalysis.h" | ||||||||
| 15 | #include "llvm/Analysis/InstructionSimplify.h" | ||||||||
| 16 | #include "llvm/IR/ConstantRange.h" | ||||||||
| 17 | #include "llvm/IR/Intrinsics.h" | ||||||||
| 18 | #include "llvm/IR/PatternMatch.h" | ||||||||
| 19 | #include "llvm/Transforms/InstCombine/InstCombiner.h" | ||||||||
| 20 | #include "llvm/Transforms/Utils/Local.h" | ||||||||
| 21 | |||||||||
| 22 | using namespace llvm; | ||||||||
| 23 | using namespace PatternMatch; | ||||||||
| 24 | |||||||||
| 25 | #define DEBUG_TYPE"instcombine" "instcombine" | ||||||||
| 26 | |||||||||
| 27 | /// This is the complement of getICmpCode, which turns an opcode and two | ||||||||
| 28 | /// operands into either a constant true or false, or a brand new ICmp | ||||||||
| 29 | /// instruction. The sign is passed in to determine which kind of predicate to | ||||||||
| 30 | /// use in the new icmp instruction. | ||||||||
| 31 | static Value *getNewICmpValue(unsigned Code, bool Sign, Value *LHS, Value *RHS, | ||||||||
| 32 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 33 | ICmpInst::Predicate NewPred; | ||||||||
| 34 | if (Constant *TorF = getPredForICmpCode(Code, Sign, LHS->getType(), NewPred)) | ||||||||
| 35 | return TorF; | ||||||||
| 36 | return Builder.CreateICmp(NewPred, LHS, RHS); | ||||||||
| 37 | } | ||||||||
| 38 | |||||||||
| 39 | /// This is the complement of getFCmpCode, which turns an opcode and two | ||||||||
| 40 | /// operands into either a FCmp instruction, or a true/false constant. | ||||||||
| 41 | static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS, | ||||||||
| 42 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 43 | FCmpInst::Predicate NewPred; | ||||||||
| 44 | if (Constant *TorF = getPredForFCmpCode(Code, LHS->getType(), NewPred)) | ||||||||
| 45 | return TorF; | ||||||||
| 46 | return Builder.CreateFCmp(NewPred, LHS, RHS); | ||||||||
| 47 | } | ||||||||
| 48 | |||||||||
| 49 | /// Transform BITWISE_OP(BSWAP(A),BSWAP(B)) or | ||||||||
| 50 | /// BITWISE_OP(BSWAP(A), Constant) to BSWAP(BITWISE_OP(A, B)) | ||||||||
| 51 | /// \param I Binary operator to transform. | ||||||||
| 52 | /// \return Pointer to node that must replace the original binary operator, or | ||||||||
| 53 | /// null pointer if no transformation was made. | ||||||||
| 54 | static Value *SimplifyBSwap(BinaryOperator &I, | ||||||||
| 55 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 56 | assert(I.isBitwiseLogicOp() && "Unexpected opcode for bswap simplifying")(static_cast <bool> (I.isBitwiseLogicOp() && "Unexpected opcode for bswap simplifying" ) ? void (0) : __assert_fail ("I.isBitwiseLogicOp() && \"Unexpected opcode for bswap simplifying\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 56 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 57 | |||||||||
| 58 | Value *OldLHS = I.getOperand(0); | ||||||||
| 59 | Value *OldRHS = I.getOperand(1); | ||||||||
| 60 | |||||||||
| 61 | Value *NewLHS; | ||||||||
| 62 | if (!match(OldLHS, m_BSwap(m_Value(NewLHS)))) | ||||||||
| 63 | return nullptr; | ||||||||
| 64 | |||||||||
| 65 | Value *NewRHS; | ||||||||
| 66 | const APInt *C; | ||||||||
| 67 | |||||||||
| 68 | if (match(OldRHS, m_BSwap(m_Value(NewRHS)))) { | ||||||||
| 69 | // OP( BSWAP(x), BSWAP(y) ) -> BSWAP( OP(x, y) ) | ||||||||
| 70 | if (!OldLHS->hasOneUse() && !OldRHS->hasOneUse()) | ||||||||
| 71 | return nullptr; | ||||||||
| 72 | // NewRHS initialized by the matcher. | ||||||||
| 73 | } else if (match(OldRHS, m_APInt(C))) { | ||||||||
| 74 | // OP( BSWAP(x), CONSTANT ) -> BSWAP( OP(x, BSWAP(CONSTANT) ) ) | ||||||||
| 75 | if (!OldLHS->hasOneUse()) | ||||||||
| 76 | return nullptr; | ||||||||
| 77 | NewRHS = ConstantInt::get(I.getType(), C->byteSwap()); | ||||||||
| 78 | } else | ||||||||
| 79 | return nullptr; | ||||||||
| 80 | |||||||||
| 81 | Value *BinOp = Builder.CreateBinOp(I.getOpcode(), NewLHS, NewRHS); | ||||||||
| 82 | Function *F = Intrinsic::getDeclaration(I.getModule(), Intrinsic::bswap, | ||||||||
| 83 | I.getType()); | ||||||||
| 84 | return Builder.CreateCall(F, BinOp); | ||||||||
| 85 | } | ||||||||
| 86 | |||||||||
| 87 | /// Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise | ||||||||
| 88 | /// (V < Lo || V >= Hi). This method expects that Lo < Hi. IsSigned indicates | ||||||||
| 89 | /// whether to treat V, Lo, and Hi as signed or not. | ||||||||
| 90 | Value *InstCombinerImpl::insertRangeTest(Value *V, const APInt &Lo, | ||||||||
| 91 | const APInt &Hi, bool isSigned, | ||||||||
| 92 | bool Inside) { | ||||||||
| 93 | assert((isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) &&(static_cast <bool> ((isSigned ? Lo.slt(Hi) : Lo.ult(Hi )) && "Lo is not < Hi in range emission code!") ? void (0) : __assert_fail ("(isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) && \"Lo is not < Hi in range emission code!\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 94 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 94 | "Lo is not < Hi in range emission code!")(static_cast <bool> ((isSigned ? Lo.slt(Hi) : Lo.ult(Hi )) && "Lo is not < Hi in range emission code!") ? void (0) : __assert_fail ("(isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) && \"Lo is not < Hi in range emission code!\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 94 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 95 | |||||||||
| 96 | Type *Ty = V->getType(); | ||||||||
| 97 | |||||||||
| 98 | // V >= Min && V < Hi --> V < Hi | ||||||||
| 99 | // V < Min || V >= Hi --> V >= Hi | ||||||||
| 100 | ICmpInst::Predicate Pred = Inside ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE; | ||||||||
| 101 | if (isSigned ? Lo.isMinSignedValue() : Lo.isMinValue()) { | ||||||||
| 102 | Pred = isSigned ? ICmpInst::getSignedPredicate(Pred) : Pred; | ||||||||
| 103 | return Builder.CreateICmp(Pred, V, ConstantInt::get(Ty, Hi)); | ||||||||
| 104 | } | ||||||||
| 105 | |||||||||
| 106 | // V >= Lo && V < Hi --> V - Lo u< Hi - Lo | ||||||||
| 107 | // V < Lo || V >= Hi --> V - Lo u>= Hi - Lo | ||||||||
| 108 | Value *VMinusLo = | ||||||||
| 109 | Builder.CreateSub(V, ConstantInt::get(Ty, Lo), V->getName() + ".off"); | ||||||||
| 110 | Constant *HiMinusLo = ConstantInt::get(Ty, Hi - Lo); | ||||||||
| 111 | return Builder.CreateICmp(Pred, VMinusLo, HiMinusLo); | ||||||||
| 112 | } | ||||||||
| 113 | |||||||||
| 114 | /// Classify (icmp eq (A & B), C) and (icmp ne (A & B), C) as matching patterns | ||||||||
| 115 | /// that can be simplified. | ||||||||
| 116 | /// One of A and B is considered the mask. The other is the value. This is | ||||||||
| 117 | /// described as the "AMask" or "BMask" part of the enum. If the enum contains | ||||||||
| 118 | /// only "Mask", then both A and B can be considered masks. If A is the mask, | ||||||||
| 119 | /// then it was proven that (A & C) == C. This is trivial if C == A or C == 0. | ||||||||
| 120 | /// If both A and C are constants, this proof is also easy. | ||||||||
| 121 | /// For the following explanations, we assume that A is the mask. | ||||||||
| 122 | /// | ||||||||
| 123 | /// "AllOnes" declares that the comparison is true only if (A & B) == A or all | ||||||||
| 124 | /// bits of A are set in B. | ||||||||
| 125 | /// Example: (icmp eq (A & 3), 3) -> AMask_AllOnes | ||||||||
| 126 | /// | ||||||||
| 127 | /// "AllZeros" declares that the comparison is true only if (A & B) == 0 or all | ||||||||
| 128 | /// bits of A are cleared in B. | ||||||||
| 129 | /// Example: (icmp eq (A & 3), 0) -> Mask_AllZeroes | ||||||||
| 130 | /// | ||||||||
| 131 | /// "Mixed" declares that (A & B) == C and C might or might not contain any | ||||||||
| 132 | /// number of one bits and zero bits. | ||||||||
| 133 | /// Example: (icmp eq (A & 3), 1) -> AMask_Mixed | ||||||||
| 134 | /// | ||||||||
| 135 | /// "Not" means that in above descriptions "==" should be replaced by "!=". | ||||||||
| 136 | /// Example: (icmp ne (A & 3), 3) -> AMask_NotAllOnes | ||||||||
| 137 | /// | ||||||||
| 138 | /// If the mask A contains a single bit, then the following is equivalent: | ||||||||
| 139 | /// (icmp eq (A & B), A) equals (icmp ne (A & B), 0) | ||||||||
| 140 | /// (icmp ne (A & B), A) equals (icmp eq (A & B), 0) | ||||||||
| 141 | enum MaskedICmpType { | ||||||||
| 142 | AMask_AllOnes = 1, | ||||||||
| 143 | AMask_NotAllOnes = 2, | ||||||||
| 144 | BMask_AllOnes = 4, | ||||||||
| 145 | BMask_NotAllOnes = 8, | ||||||||
| 146 | Mask_AllZeros = 16, | ||||||||
| 147 | Mask_NotAllZeros = 32, | ||||||||
| 148 | AMask_Mixed = 64, | ||||||||
| 149 | AMask_NotMixed = 128, | ||||||||
| 150 | BMask_Mixed = 256, | ||||||||
| 151 | BMask_NotMixed = 512 | ||||||||
| 152 | }; | ||||||||
| 153 | |||||||||
| 154 | /// Return the set of patterns (from MaskedICmpType) that (icmp SCC (A & B), C) | ||||||||
| 155 | /// satisfies. | ||||||||
| 156 | static unsigned getMaskedICmpType(Value *A, Value *B, Value *C, | ||||||||
| 157 | ICmpInst::Predicate Pred) { | ||||||||
| 158 | const APInt *ConstA = nullptr, *ConstB = nullptr, *ConstC = nullptr; | ||||||||
| 159 | match(A, m_APInt(ConstA)); | ||||||||
| 160 | match(B, m_APInt(ConstB)); | ||||||||
| 161 | match(C, m_APInt(ConstC)); | ||||||||
| 162 | bool IsEq = (Pred == ICmpInst::ICMP_EQ); | ||||||||
| 163 | bool IsAPow2 = ConstA && ConstA->isPowerOf2(); | ||||||||
| 164 | bool IsBPow2 = ConstB && ConstB->isPowerOf2(); | ||||||||
| 165 | unsigned MaskVal = 0; | ||||||||
| 166 | if (ConstC && ConstC->isZero()) { | ||||||||
| 167 | // if C is zero, then both A and B qualify as mask | ||||||||
| 168 | MaskVal |= (IsEq ? (Mask_AllZeros | AMask_Mixed | BMask_Mixed) | ||||||||
| 169 | : (Mask_NotAllZeros | AMask_NotMixed | BMask_NotMixed)); | ||||||||
| 170 | if (IsAPow2) | ||||||||
| 171 | MaskVal |= (IsEq ? (AMask_NotAllOnes | AMask_NotMixed) | ||||||||
| 172 | : (AMask_AllOnes | AMask_Mixed)); | ||||||||
| 173 | if (IsBPow2) | ||||||||
| 174 | MaskVal |= (IsEq ? (BMask_NotAllOnes | BMask_NotMixed) | ||||||||
| 175 | : (BMask_AllOnes | BMask_Mixed)); | ||||||||
| 176 | return MaskVal; | ||||||||
| 177 | } | ||||||||
| 178 | |||||||||
| 179 | if (A == C) { | ||||||||
| 180 | MaskVal |= (IsEq ? (AMask_AllOnes | AMask_Mixed) | ||||||||
| 181 | : (AMask_NotAllOnes | AMask_NotMixed)); | ||||||||
| 182 | if (IsAPow2) | ||||||||
| 183 | MaskVal |= (IsEq ? (Mask_NotAllZeros | AMask_NotMixed) | ||||||||
| 184 | : (Mask_AllZeros | AMask_Mixed)); | ||||||||
| 185 | } else if (ConstA && ConstC && ConstC->isSubsetOf(*ConstA)) { | ||||||||
| 186 | MaskVal |= (IsEq ? AMask_Mixed : AMask_NotMixed); | ||||||||
| 187 | } | ||||||||
| 188 | |||||||||
| 189 | if (B == C) { | ||||||||
| 190 | MaskVal |= (IsEq ? (BMask_AllOnes | BMask_Mixed) | ||||||||
| 191 | : (BMask_NotAllOnes | BMask_NotMixed)); | ||||||||
| 192 | if (IsBPow2) | ||||||||
| 193 | MaskVal |= (IsEq ? (Mask_NotAllZeros | BMask_NotMixed) | ||||||||
| 194 | : (Mask_AllZeros | BMask_Mixed)); | ||||||||
| 195 | } else if (ConstB && ConstC && ConstC->isSubsetOf(*ConstB)) { | ||||||||
| 196 | MaskVal |= (IsEq ? BMask_Mixed : BMask_NotMixed); | ||||||||
| 197 | } | ||||||||
| 198 | |||||||||
| 199 | return MaskVal; | ||||||||
| 200 | } | ||||||||
| 201 | |||||||||
| 202 | /// Convert an analysis of a masked ICmp into its equivalent if all boolean | ||||||||
| 203 | /// operations had the opposite sense. Since each "NotXXX" flag (recording !=) | ||||||||
| 204 | /// is adjacent to the corresponding normal flag (recording ==), this just | ||||||||
| 205 | /// involves swapping those bits over. | ||||||||
| 206 | static unsigned conjugateICmpMask(unsigned Mask) { | ||||||||
| 207 | unsigned NewMask; | ||||||||
| 208 | NewMask = (Mask & (AMask_AllOnes | BMask_AllOnes | Mask_AllZeros | | ||||||||
| 209 | AMask_Mixed | BMask_Mixed)) | ||||||||
| 210 | << 1; | ||||||||
| 211 | |||||||||
| 212 | NewMask |= (Mask & (AMask_NotAllOnes | BMask_NotAllOnes | Mask_NotAllZeros | | ||||||||
| 213 | AMask_NotMixed | BMask_NotMixed)) | ||||||||
| 214 | >> 1; | ||||||||
| 215 | |||||||||
| 216 | return NewMask; | ||||||||
| 217 | } | ||||||||
| 218 | |||||||||
| 219 | // Adapts the external decomposeBitTestICmp for local use. | ||||||||
| 220 | static bool decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate &Pred, | ||||||||
| 221 | Value *&X, Value *&Y, Value *&Z) { | ||||||||
| 222 | APInt Mask; | ||||||||
| 223 | if (!llvm::decomposeBitTestICmp(LHS, RHS, Pred, X, Mask)) | ||||||||
| 224 | return false; | ||||||||
| 225 | |||||||||
| 226 | Y = ConstantInt::get(X->getType(), Mask); | ||||||||
| 227 | Z = ConstantInt::get(X->getType(), 0); | ||||||||
| 228 | return true; | ||||||||
| 229 | } | ||||||||
| 230 | |||||||||
| 231 | /// Handle (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E). | ||||||||
| 232 | /// Return the pattern classes (from MaskedICmpType) for the left hand side and | ||||||||
| 233 | /// the right hand side as a pair. | ||||||||
| 234 | /// LHS and RHS are the left hand side and the right hand side ICmps and PredL | ||||||||
| 235 | /// and PredR are their predicates, respectively. | ||||||||
| 236 | static std::optional<std::pair<unsigned, unsigned>> getMaskedTypeForICmpPair( | ||||||||
| 237 | Value *&A, Value *&B, Value *&C, Value *&D, Value *&E, ICmpInst *LHS, | ||||||||
| 238 | ICmpInst *RHS, ICmpInst::Predicate &PredL, ICmpInst::Predicate &PredR) { | ||||||||
| 239 | // Don't allow pointers. Splat vectors are fine. | ||||||||
| 240 | if (!LHS->getOperand(0)->getType()->isIntOrIntVectorTy() || | ||||||||
| 241 | !RHS->getOperand(0)->getType()->isIntOrIntVectorTy()) | ||||||||
| 242 | return std::nullopt; | ||||||||
| 243 | |||||||||
| 244 | // Here comes the tricky part: | ||||||||
| 245 | // LHS might be of the form L11 & L12 == X, X == L21 & L22, | ||||||||
| 246 | // and L11 & L12 == L21 & L22. The same goes for RHS. | ||||||||
| 247 | // Now we must find those components L** and R**, that are equal, so | ||||||||
| 248 | // that we can extract the parameters A, B, C, D, and E for the canonical | ||||||||
| 249 | // above. | ||||||||
| 250 | Value *L1 = LHS->getOperand(0); | ||||||||
| 251 | Value *L2 = LHS->getOperand(1); | ||||||||
| 252 | Value *L11, *L12, *L21, *L22; | ||||||||
| 253 | // Check whether the icmp can be decomposed into a bit test. | ||||||||
| 254 | if (decomposeBitTestICmp(L1, L2, PredL, L11, L12, L2)) { | ||||||||
| 255 | L21 = L22 = L1 = nullptr; | ||||||||
| 256 | } else { | ||||||||
| 257 | // Look for ANDs in the LHS icmp. | ||||||||
| 258 | if (!match(L1, m_And(m_Value(L11), m_Value(L12)))) { | ||||||||
| 259 | // Any icmp can be viewed as being trivially masked; if it allows us to | ||||||||
| 260 | // remove one, it's worth it. | ||||||||
| 261 | L11 = L1; | ||||||||
| 262 | L12 = Constant::getAllOnesValue(L1->getType()); | ||||||||
| 263 | } | ||||||||
| 264 | |||||||||
| 265 | if (!match(L2, m_And(m_Value(L21), m_Value(L22)))) { | ||||||||
| 266 | L21 = L2; | ||||||||
| 267 | L22 = Constant::getAllOnesValue(L2->getType()); | ||||||||
| 268 | } | ||||||||
| 269 | } | ||||||||
| 270 | |||||||||
| 271 | // Bail if LHS was a icmp that can't be decomposed into an equality. | ||||||||
| 272 | if (!ICmpInst::isEquality(PredL)) | ||||||||
| 273 | return std::nullopt; | ||||||||
| 274 | |||||||||
| 275 | Value *R1 = RHS->getOperand(0); | ||||||||
| 276 | Value *R2 = RHS->getOperand(1); | ||||||||
| 277 | Value *R11, *R12; | ||||||||
| 278 | bool Ok = false; | ||||||||
| 279 | if (decomposeBitTestICmp(R1, R2, PredR, R11, R12, R2)) { | ||||||||
| 280 | if (R11 == L11 || R11 == L12 || R11
| ||||||||
| 281 | A = R11; | ||||||||
| 282 | D = R12; | ||||||||
| 283 | } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) { | ||||||||
| 284 | A = R12; | ||||||||
| 285 | D = R11; | ||||||||
| 286 | } else { | ||||||||
| 287 | return std::nullopt; | ||||||||
| 288 | } | ||||||||
| 289 | E = R2; | ||||||||
| 290 | R1 = nullptr; | ||||||||
| 291 | Ok = true; | ||||||||
| 292 | } else { | ||||||||
| 293 | if (!match(R1, m_And(m_Value(R11), m_Value(R12)))) { | ||||||||
| 294 | // As before, model no mask as a trivial mask if it'll let us do an | ||||||||
| 295 | // optimization. | ||||||||
| 296 | R11 = R1; | ||||||||
| 297 | R12 = Constant::getAllOnesValue(R1->getType()); | ||||||||
| 298 | } | ||||||||
| 299 | |||||||||
| 300 | if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) { | ||||||||
| 301 | A = R11; | ||||||||
| 302 | D = R12; | ||||||||
| 303 | E = R2; | ||||||||
| 304 | Ok = true; | ||||||||
| 305 | } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) { | ||||||||
| 306 | A = R12; | ||||||||
| 307 | D = R11; | ||||||||
| 308 | E = R2; | ||||||||
| 309 | Ok = true; | ||||||||
| 310 | } | ||||||||
| 311 | } | ||||||||
| 312 | |||||||||
| 313 | // Bail if RHS was a icmp that can't be decomposed into an equality. | ||||||||
| 314 | if (!ICmpInst::isEquality(PredR)) | ||||||||
| 315 | return std::nullopt; | ||||||||
| 316 | |||||||||
| 317 | // Look for ANDs on the right side of the RHS icmp. | ||||||||
| 318 | if (!Ok
| ||||||||
| 319 | if (!match(R2, m_And(m_Value(R11), m_Value(R12)))) { | ||||||||
| 320 | R11 = R2; | ||||||||
| 321 | R12 = Constant::getAllOnesValue(R2->getType()); | ||||||||
| 322 | } | ||||||||
| 323 | |||||||||
| 324 | if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) { | ||||||||
| 325 | A = R11; | ||||||||
| 326 | D = R12; | ||||||||
| 327 | E = R1; | ||||||||
| 328 | Ok = true; | ||||||||
| 329 | } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) { | ||||||||
| 330 | A = R12; | ||||||||
| 331 | D = R11; | ||||||||
| 332 | E = R1; | ||||||||
| 333 | Ok = true; | ||||||||
| 334 | } else { | ||||||||
| 335 | return std::nullopt; | ||||||||
| 336 | } | ||||||||
| 337 | |||||||||
| 338 | assert(Ok && "Failed to find AND on the right side of the RHS icmp.")(static_cast <bool> (Ok && "Failed to find AND on the right side of the RHS icmp." ) ? void (0) : __assert_fail ("Ok && \"Failed to find AND on the right side of the RHS icmp.\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 338 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 339 | } | ||||||||
| 340 | |||||||||
| 341 | if (L11
| ||||||||
| 342 | B = L12; | ||||||||
| 343 | C = L2; | ||||||||
| 344 | } else if (L12
| ||||||||
| 345 | B = L11; | ||||||||
| 346 | C = L2; | ||||||||
| 347 | } else if (L21
| ||||||||
| 348 | B = L22; | ||||||||
| 349 | C = L1; | ||||||||
| 350 | } else if (L22 == A) { | ||||||||
| 351 | B = L21; | ||||||||
| 352 | C = L1; | ||||||||
| 353 | } | ||||||||
| 354 | |||||||||
| 355 | unsigned LeftType = getMaskedICmpType(A, B, C, PredL); | ||||||||
| 356 | unsigned RightType = getMaskedICmpType(A, D, E, PredR); | ||||||||
| 357 | return std::optional<std::pair<unsigned, unsigned>>( | ||||||||
| 358 | std::make_pair(LeftType, RightType)); | ||||||||
| 359 | } | ||||||||
| 360 | |||||||||
| 361 | /// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) into a single | ||||||||
| 362 | /// (icmp(A & X) ==/!= Y), where the left-hand side is of type Mask_NotAllZeros | ||||||||
| 363 | /// and the right hand side is of type BMask_Mixed. For example, | ||||||||
| 364 | /// (icmp (A & 12) != 0) & (icmp (A & 15) == 8) -> (icmp (A & 15) == 8). | ||||||||
| 365 | /// Also used for logical and/or, must be poison safe. | ||||||||
| 366 | static Value *foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed( | ||||||||
| 367 | ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, Value *A, Value *B, Value *C, | ||||||||
| 368 | Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR, | ||||||||
| 369 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 370 | // We are given the canonical form: | ||||||||
| 371 | // (icmp ne (A & B), 0) & (icmp eq (A & D), E). | ||||||||
| 372 | // where D & E == E. | ||||||||
| 373 | // | ||||||||
| 374 | // If IsAnd is false, we get it in negated form: | ||||||||
| 375 | // (icmp eq (A & B), 0) | (icmp ne (A & D), E) -> | ||||||||
| 376 | // !((icmp ne (A & B), 0) & (icmp eq (A & D), E)). | ||||||||
| 377 | // | ||||||||
| 378 | // We currently handle the case of B, C, D, E are constant. | ||||||||
| 379 | // | ||||||||
| 380 | const APInt *BCst, *CCst, *DCst, *OrigECst; | ||||||||
| 381 | if (!match(B, m_APInt(BCst)) || !match(C, m_APInt(CCst)) || | ||||||||
| 382 | !match(D, m_APInt(DCst)) || !match(E, m_APInt(OrigECst))) | ||||||||
| 383 | return nullptr; | ||||||||
| 384 | |||||||||
| 385 | ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE; | ||||||||
| 386 | |||||||||
| 387 | // Update E to the canonical form when D is a power of two and RHS is | ||||||||
| 388 | // canonicalized as, | ||||||||
| 389 | // (icmp ne (A & D), 0) -> (icmp eq (A & D), D) or | ||||||||
| 390 | // (icmp ne (A & D), D) -> (icmp eq (A & D), 0). | ||||||||
| 391 | APInt ECst = *OrigECst; | ||||||||
| 392 | if (PredR != NewCC) | ||||||||
| 393 | ECst ^= *DCst; | ||||||||
| 394 | |||||||||
| 395 | // If B or D is zero, skip because if LHS or RHS can be trivially folded by | ||||||||
| 396 | // other folding rules and this pattern won't apply any more. | ||||||||
| 397 | if (*BCst == 0 || *DCst == 0) | ||||||||
| 398 | return nullptr; | ||||||||
| 399 | |||||||||
| 400 | // If B and D don't intersect, ie. (B & D) == 0, no folding because we can't | ||||||||
| 401 | // deduce anything from it. | ||||||||
| 402 | // For example, | ||||||||
| 403 | // (icmp ne (A & 12), 0) & (icmp eq (A & 3), 1) -> no folding. | ||||||||
| 404 | if ((*BCst & *DCst) == 0) | ||||||||
| 405 | return nullptr; | ||||||||
| 406 | |||||||||
| 407 | // If the following two conditions are met: | ||||||||
| 408 | // | ||||||||
| 409 | // 1. mask B covers only a single bit that's not covered by mask D, that is, | ||||||||
| 410 | // (B & (B ^ D)) is a power of 2 (in other words, B minus the intersection of | ||||||||
| 411 | // B and D has only one bit set) and, | ||||||||
| 412 | // | ||||||||
| 413 | // 2. RHS (and E) indicates that the rest of B's bits are zero (in other | ||||||||
| 414 | // words, the intersection of B and D is zero), that is, ((B & D) & E) == 0 | ||||||||
| 415 | // | ||||||||
| 416 | // then that single bit in B must be one and thus the whole expression can be | ||||||||
| 417 | // folded to | ||||||||
| 418 | // (A & (B | D)) == (B & (B ^ D)) | E. | ||||||||
| 419 | // | ||||||||
| 420 | // For example, | ||||||||
| 421 | // (icmp ne (A & 12), 0) & (icmp eq (A & 7), 1) -> (icmp eq (A & 15), 9) | ||||||||
| 422 | // (icmp ne (A & 15), 0) & (icmp eq (A & 7), 0) -> (icmp eq (A & 15), 8) | ||||||||
| 423 | if ((((*BCst & *DCst) & ECst) == 0) && | ||||||||
| 424 | (*BCst & (*BCst ^ *DCst)).isPowerOf2()) { | ||||||||
| 425 | APInt BorD = *BCst | *DCst; | ||||||||
| 426 | APInt BandBxorDorE = (*BCst & (*BCst ^ *DCst)) | ECst; | ||||||||
| 427 | Value *NewMask = ConstantInt::get(A->getType(), BorD); | ||||||||
| 428 | Value *NewMaskedValue = ConstantInt::get(A->getType(), BandBxorDorE); | ||||||||
| 429 | Value *NewAnd = Builder.CreateAnd(A, NewMask); | ||||||||
| 430 | return Builder.CreateICmp(NewCC, NewAnd, NewMaskedValue); | ||||||||
| 431 | } | ||||||||
| 432 | |||||||||
| 433 | auto IsSubSetOrEqual = [](const APInt *C1, const APInt *C2) { | ||||||||
| 434 | return (*C1 & *C2) == *C1; | ||||||||
| 435 | }; | ||||||||
| 436 | auto IsSuperSetOrEqual = [](const APInt *C1, const APInt *C2) { | ||||||||
| 437 | return (*C1 & *C2) == *C2; | ||||||||
| 438 | }; | ||||||||
| 439 | |||||||||
| 440 | // In the following, we consider only the cases where B is a superset of D, B | ||||||||
| 441 | // is a subset of D, or B == D because otherwise there's at least one bit | ||||||||
| 442 | // covered by B but not D, in which case we can't deduce much from it, so | ||||||||
| 443 | // no folding (aside from the single must-be-one bit case right above.) | ||||||||
| 444 | // For example, | ||||||||
| 445 | // (icmp ne (A & 14), 0) & (icmp eq (A & 3), 1) -> no folding. | ||||||||
| 446 | if (!IsSubSetOrEqual(BCst, DCst) && !IsSuperSetOrEqual(BCst, DCst)) | ||||||||
| 447 | return nullptr; | ||||||||
| 448 | |||||||||
| 449 | // At this point, either B is a superset of D, B is a subset of D or B == D. | ||||||||
| 450 | |||||||||
| 451 | // If E is zero, if B is a subset of (or equal to) D, LHS and RHS contradict | ||||||||
| 452 | // and the whole expression becomes false (or true if negated), otherwise, no | ||||||||
| 453 | // folding. | ||||||||
| 454 | // For example, | ||||||||
| 455 | // (icmp ne (A & 3), 0) & (icmp eq (A & 7), 0) -> false. | ||||||||
| 456 | // (icmp ne (A & 15), 0) & (icmp eq (A & 3), 0) -> no folding. | ||||||||
| 457 | if (ECst.isZero()) { | ||||||||
| 458 | if (IsSubSetOrEqual(BCst, DCst)) | ||||||||
| 459 | return ConstantInt::get(LHS->getType(), !IsAnd); | ||||||||
| 460 | return nullptr; | ||||||||
| 461 | } | ||||||||
| 462 | |||||||||
| 463 | // At this point, B, D, E aren't zero and (B & D) == B, (B & D) == D or B == | ||||||||
| 464 | // D. If B is a superset of (or equal to) D, since E is not zero, LHS is | ||||||||
| 465 | // subsumed by RHS (RHS implies LHS.) So the whole expression becomes | ||||||||
| 466 | // RHS. For example, | ||||||||
| 467 | // (icmp ne (A & 255), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8). | ||||||||
| 468 | // (icmp ne (A & 15), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8). | ||||||||
| 469 | if (IsSuperSetOrEqual(BCst, DCst)) | ||||||||
| 470 | return RHS; | ||||||||
| 471 | // Otherwise, B is a subset of D. If B and E have a common bit set, | ||||||||
| 472 | // ie. (B & E) != 0, then LHS is subsumed by RHS. For example. | ||||||||
| 473 | // (icmp ne (A & 12), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8). | ||||||||
| 474 | assert(IsSubSetOrEqual(BCst, DCst) && "Precondition due to above code")(static_cast <bool> (IsSubSetOrEqual(BCst, DCst) && "Precondition due to above code") ? void (0) : __assert_fail ("IsSubSetOrEqual(BCst, DCst) && \"Precondition due to above code\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 474 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 475 | if ((*BCst & ECst) != 0) | ||||||||
| 476 | return RHS; | ||||||||
| 477 | // Otherwise, LHS and RHS contradict and the whole expression becomes false | ||||||||
| 478 | // (or true if negated.) For example, | ||||||||
| 479 | // (icmp ne (A & 7), 0) & (icmp eq (A & 15), 8) -> false. | ||||||||
| 480 | // (icmp ne (A & 6), 0) & (icmp eq (A & 15), 8) -> false. | ||||||||
| 481 | return ConstantInt::get(LHS->getType(), !IsAnd); | ||||||||
| 482 | } | ||||||||
| 483 | |||||||||
| 484 | /// Try to fold (icmp(A & B) ==/!= 0) &/| (icmp(A & D) ==/!= E) into a single | ||||||||
| 485 | /// (icmp(A & X) ==/!= Y), where the left-hand side and the right hand side | ||||||||
| 486 | /// aren't of the common mask pattern type. | ||||||||
| 487 | /// Also used for logical and/or, must be poison safe. | ||||||||
| 488 | static Value *foldLogOpOfMaskedICmpsAsymmetric( | ||||||||
| 489 | ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, Value *A, Value *B, Value *C, | ||||||||
| 490 | Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR, | ||||||||
| 491 | unsigned LHSMask, unsigned RHSMask, InstCombiner::BuilderTy &Builder) { | ||||||||
| 492 | assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&(static_cast <bool> (ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && "Expected equality predicates for masked type of icmps." ) ? void (0) : __assert_fail ("ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && \"Expected equality predicates for masked type of icmps.\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 493 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 493 | "Expected equality predicates for masked type of icmps.")(static_cast <bool> (ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && "Expected equality predicates for masked type of icmps." ) ? void (0) : __assert_fail ("ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && \"Expected equality predicates for masked type of icmps.\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 493 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 494 | // Handle Mask_NotAllZeros-BMask_Mixed cases. | ||||||||
| 495 | // (icmp ne/eq (A & B), C) &/| (icmp eq/ne (A & D), E), or | ||||||||
| 496 | // (icmp eq/ne (A & B), C) &/| (icmp ne/eq (A & D), E) | ||||||||
| 497 | // which gets swapped to | ||||||||
| 498 | // (icmp ne/eq (A & D), E) &/| (icmp eq/ne (A & B), C). | ||||||||
| 499 | if (!IsAnd) { | ||||||||
| 500 | LHSMask = conjugateICmpMask(LHSMask); | ||||||||
| 501 | RHSMask = conjugateICmpMask(RHSMask); | ||||||||
| 502 | } | ||||||||
| 503 | if ((LHSMask & Mask_NotAllZeros) && (RHSMask & BMask_Mixed)) { | ||||||||
| 504 | if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed( | ||||||||
| 505 | LHS, RHS, IsAnd, A, B, C, D, E, | ||||||||
| 506 | PredL, PredR, Builder)) { | ||||||||
| 507 | return V; | ||||||||
| 508 | } | ||||||||
| 509 | } else if ((LHSMask & BMask_Mixed) && (RHSMask & Mask_NotAllZeros)) { | ||||||||
| 510 | if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed( | ||||||||
| 511 | RHS, LHS, IsAnd, A, D, E, B, C, | ||||||||
| 512 | PredR, PredL, Builder)) { | ||||||||
| 513 | return V; | ||||||||
| 514 | } | ||||||||
| 515 | } | ||||||||
| 516 | return nullptr; | ||||||||
| 517 | } | ||||||||
| 518 | |||||||||
| 519 | /// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) | ||||||||
| 520 | /// into a single (icmp(A & X) ==/!= Y). | ||||||||
| 521 | static Value *foldLogOpOfMaskedICmps(ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, | ||||||||
| 522 | bool IsLogical, | ||||||||
| 523 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 524 | Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr; | ||||||||
| 525 | ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate(); | ||||||||
| 526 | std::optional<std::pair<unsigned, unsigned>> MaskPair = | ||||||||
| 527 | getMaskedTypeForICmpPair(A, B, C, D, E, LHS, RHS, PredL, PredR); | ||||||||
| 528 | if (!MaskPair) | ||||||||
| 529 | return nullptr; | ||||||||
| 530 | assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&(static_cast <bool> (ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && "Expected equality predicates for masked type of icmps." ) ? void (0) : __assert_fail ("ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && \"Expected equality predicates for masked type of icmps.\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 531 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 531 | "Expected equality predicates for masked type of icmps.")(static_cast <bool> (ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && "Expected equality predicates for masked type of icmps." ) ? void (0) : __assert_fail ("ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) && \"Expected equality predicates for masked type of icmps.\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 531 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 532 | unsigned LHSMask = MaskPair->first; | ||||||||
| 533 | unsigned RHSMask = MaskPair->second; | ||||||||
| 534 | unsigned Mask = LHSMask & RHSMask; | ||||||||
| 535 | if (Mask == 0) { | ||||||||
| 536 | // Even if the two sides don't share a common pattern, check if folding can | ||||||||
| 537 | // still happen. | ||||||||
| 538 | if (Value *V = foldLogOpOfMaskedICmpsAsymmetric( | ||||||||
| 539 | LHS, RHS, IsAnd, A, B, C, D, E, PredL, PredR, LHSMask, RHSMask, | ||||||||
| 540 | Builder)) | ||||||||
| 541 | return V; | ||||||||
| 542 | return nullptr; | ||||||||
| 543 | } | ||||||||
| 544 | |||||||||
| 545 | // In full generality: | ||||||||
| 546 | // (icmp (A & B) Op C) | (icmp (A & D) Op E) | ||||||||
| 547 | // == ![ (icmp (A & B) !Op C) & (icmp (A & D) !Op E) ] | ||||||||
| 548 | // | ||||||||
| 549 | // If the latter can be converted into (icmp (A & X) Op Y) then the former is | ||||||||
| 550 | // equivalent to (icmp (A & X) !Op Y). | ||||||||
| 551 | // | ||||||||
| 552 | // Therefore, we can pretend for the rest of this function that we're dealing | ||||||||
| 553 | // with the conjunction, provided we flip the sense of any comparisons (both | ||||||||
| 554 | // input and output). | ||||||||
| 555 | |||||||||
| 556 | // In most cases we're going to produce an EQ for the "&&" case. | ||||||||
| 557 | ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE; | ||||||||
| 558 | if (!IsAnd) { | ||||||||
| 559 | // Convert the masking analysis into its equivalent with negated | ||||||||
| 560 | // comparisons. | ||||||||
| 561 | Mask = conjugateICmpMask(Mask); | ||||||||
| 562 | } | ||||||||
| 563 | |||||||||
| 564 | if (Mask & Mask_AllZeros) { | ||||||||
| 565 | // (icmp eq (A & B), 0) & (icmp eq (A & D), 0) | ||||||||
| 566 | // -> (icmp eq (A & (B|D)), 0) | ||||||||
| 567 | if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(D)) | ||||||||
| 568 | return nullptr; // TODO: Use freeze? | ||||||||
| 569 | Value *NewOr = Builder.CreateOr(B, D); | ||||||||
| 570 | Value *NewAnd = Builder.CreateAnd(A, NewOr); | ||||||||
| 571 | // We can't use C as zero because we might actually handle | ||||||||
| 572 | // (icmp ne (A & B), B) & (icmp ne (A & D), D) | ||||||||
| 573 | // with B and D, having a single bit set. | ||||||||
| 574 | Value *Zero = Constant::getNullValue(A->getType()); | ||||||||
| 575 | return Builder.CreateICmp(NewCC, NewAnd, Zero); | ||||||||
| 576 | } | ||||||||
| 577 | if (Mask & BMask_AllOnes) { | ||||||||
| 578 | // (icmp eq (A & B), B) & (icmp eq (A & D), D) | ||||||||
| 579 | // -> (icmp eq (A & (B|D)), (B|D)) | ||||||||
| 580 | if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(D)) | ||||||||
| 581 | return nullptr; // TODO: Use freeze? | ||||||||
| 582 | Value *NewOr = Builder.CreateOr(B, D); | ||||||||
| 583 | Value *NewAnd = Builder.CreateAnd(A, NewOr); | ||||||||
| 584 | return Builder.CreateICmp(NewCC, NewAnd, NewOr); | ||||||||
| 585 | } | ||||||||
| 586 | if (Mask & AMask_AllOnes) { | ||||||||
| 587 | // (icmp eq (A & B), A) & (icmp eq (A & D), A) | ||||||||
| 588 | // -> (icmp eq (A & (B&D)), A) | ||||||||
| 589 | if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(D)) | ||||||||
| 590 | return nullptr; // TODO: Use freeze? | ||||||||
| 591 | Value *NewAnd1 = Builder.CreateAnd(B, D); | ||||||||
| 592 | Value *NewAnd2 = Builder.CreateAnd(A, NewAnd1); | ||||||||
| 593 | return Builder.CreateICmp(NewCC, NewAnd2, A); | ||||||||
| 594 | } | ||||||||
| 595 | |||||||||
| 596 | // Remaining cases assume at least that B and D are constant, and depend on | ||||||||
| 597 | // their actual values. This isn't strictly necessary, just a "handle the | ||||||||
| 598 | // easy cases for now" decision. | ||||||||
| 599 | const APInt *ConstB, *ConstD; | ||||||||
| 600 | if (!match(B, m_APInt(ConstB)) || !match(D, m_APInt(ConstD))) | ||||||||
| 601 | return nullptr; | ||||||||
| 602 | |||||||||
| 603 | if (Mask & (Mask_NotAllZeros | BMask_NotAllOnes)) { | ||||||||
| 604 | // (icmp ne (A & B), 0) & (icmp ne (A & D), 0) and | ||||||||
| 605 | // (icmp ne (A & B), B) & (icmp ne (A & D), D) | ||||||||
| 606 | // -> (icmp ne (A & B), 0) or (icmp ne (A & D), 0) | ||||||||
| 607 | // Only valid if one of the masks is a superset of the other (check "B&D" is | ||||||||
| 608 | // the same as either B or D). | ||||||||
| 609 | APInt NewMask = *ConstB & *ConstD; | ||||||||
| 610 | if (NewMask == *ConstB) | ||||||||
| 611 | return LHS; | ||||||||
| 612 | else if (NewMask == *ConstD) | ||||||||
| 613 | return RHS; | ||||||||
| 614 | } | ||||||||
| 615 | |||||||||
| 616 | if (Mask & AMask_NotAllOnes) { | ||||||||
| 617 | // (icmp ne (A & B), B) & (icmp ne (A & D), D) | ||||||||
| 618 | // -> (icmp ne (A & B), A) or (icmp ne (A & D), A) | ||||||||
| 619 | // Only valid if one of the masks is a superset of the other (check "B|D" is | ||||||||
| 620 | // the same as either B or D). | ||||||||
| 621 | APInt NewMask = *ConstB | *ConstD; | ||||||||
| 622 | if (NewMask == *ConstB) | ||||||||
| 623 | return LHS; | ||||||||
| 624 | else if (NewMask == *ConstD) | ||||||||
| 625 | return RHS; | ||||||||
| 626 | } | ||||||||
| 627 | |||||||||
| 628 | if (Mask & (BMask_Mixed | BMask_NotMixed)) { | ||||||||
| 629 | // Mixed: | ||||||||
| 630 | // (icmp eq (A & B), C) & (icmp eq (A & D), E) | ||||||||
| 631 | // We already know that B & C == C && D & E == E. | ||||||||
| 632 | // If we can prove that (B & D) & (C ^ E) == 0, that is, the bits of | ||||||||
| 633 | // C and E, which are shared by both the mask B and the mask D, don't | ||||||||
| 634 | // contradict, then we can transform to | ||||||||
| 635 | // -> (icmp eq (A & (B|D)), (C|E)) | ||||||||
| 636 | // Currently, we only handle the case of B, C, D, and E being constant. | ||||||||
| 637 | // We can't simply use C and E because we might actually handle | ||||||||
| 638 | // (icmp ne (A & B), B) & (icmp eq (A & D), D) | ||||||||
| 639 | // with B and D, having a single bit set. | ||||||||
| 640 | |||||||||
| 641 | // NotMixed: | ||||||||
| 642 | // (icmp ne (A & B), C) & (icmp ne (A & D), E) | ||||||||
| 643 | // -> (icmp ne (A & (B & D)), (C & E)) | ||||||||
| 644 | // Check the intersection (B & D) for inequality. | ||||||||
| 645 | // Assume that (B & D) == B || (B & D) == D, i.e B/D is a subset of D/B | ||||||||
| 646 | // and (B & D) & (C ^ E) == 0, bits of C and E, which are shared by both the | ||||||||
| 647 | // B and the D, don't contradict. | ||||||||
| 648 | // Note that we can assume (~B & C) == 0 && (~D & E) == 0, previous | ||||||||
| 649 | // operation should delete these icmps if it hadn't been met. | ||||||||
| 650 | |||||||||
| 651 | const APInt *OldConstC, *OldConstE; | ||||||||
| 652 | if (!match(C, m_APInt(OldConstC)) || !match(E, m_APInt(OldConstE))) | ||||||||
| 653 | return nullptr; | ||||||||
| 654 | |||||||||
| 655 | auto FoldBMixed = [&](ICmpInst::Predicate CC, bool IsNot) -> Value * { | ||||||||
| 656 | CC = IsNot ? CmpInst::getInversePredicate(CC) : CC; | ||||||||
| 657 | const APInt ConstC = PredL != CC ? *ConstB ^ *OldConstC : *OldConstC; | ||||||||
| 658 | const APInt ConstE = PredR != CC ? *ConstD ^ *OldConstE : *OldConstE; | ||||||||
| 659 | |||||||||
| 660 | if (((*ConstB & *ConstD) & (ConstC ^ ConstE)).getBoolValue()) | ||||||||
| 661 | return IsNot ? nullptr : ConstantInt::get(LHS->getType(), !IsAnd); | ||||||||
| 662 | |||||||||
| 663 | if (IsNot && !ConstB->isSubsetOf(*ConstD) && !ConstD->isSubsetOf(*ConstB)) | ||||||||
| 664 | return nullptr; | ||||||||
| 665 | |||||||||
| 666 | APInt BD, CE; | ||||||||
| 667 | if (IsNot) { | ||||||||
| 668 | BD = *ConstB & *ConstD; | ||||||||
| 669 | CE = ConstC & ConstE; | ||||||||
| 670 | } else { | ||||||||
| 671 | BD = *ConstB | *ConstD; | ||||||||
| 672 | CE = ConstC | ConstE; | ||||||||
| 673 | } | ||||||||
| 674 | Value *NewAnd = Builder.CreateAnd(A, BD); | ||||||||
| 675 | Value *CEVal = ConstantInt::get(A->getType(), CE); | ||||||||
| 676 | return Builder.CreateICmp(CC, CEVal, NewAnd); | ||||||||
| 677 | }; | ||||||||
| 678 | |||||||||
| 679 | if (Mask & BMask_Mixed) | ||||||||
| 680 | return FoldBMixed(NewCC, false); | ||||||||
| 681 | if (Mask & BMask_NotMixed) // can be else also | ||||||||
| 682 | return FoldBMixed(NewCC, true); | ||||||||
| 683 | } | ||||||||
| 684 | return nullptr; | ||||||||
| 685 | } | ||||||||
| 686 | |||||||||
| 687 | /// Try to fold a signed range checked with lower bound 0 to an unsigned icmp. | ||||||||
| 688 | /// Example: (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n | ||||||||
| 689 | /// If \p Inverted is true then the check is for the inverted range, e.g. | ||||||||
| 690 | /// (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n | ||||||||
| 691 | Value *InstCombinerImpl::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1, | ||||||||
| 692 | bool Inverted) { | ||||||||
| 693 | // Check the lower range comparison, e.g. x >= 0 | ||||||||
| 694 | // InstCombine already ensured that if there is a constant it's on the RHS. | ||||||||
| 695 | ConstantInt *RangeStart = dyn_cast<ConstantInt>(Cmp0->getOperand(1)); | ||||||||
| 696 | if (!RangeStart) | ||||||||
| 697 | return nullptr; | ||||||||
| 698 | |||||||||
| 699 | ICmpInst::Predicate Pred0 = (Inverted ? Cmp0->getInversePredicate() : | ||||||||
| 700 | Cmp0->getPredicate()); | ||||||||
| 701 | |||||||||
| 702 | // Accept x > -1 or x >= 0 (after potentially inverting the predicate). | ||||||||
| 703 | if (!((Pred0 == ICmpInst::ICMP_SGT && RangeStart->isMinusOne()) || | ||||||||
| 704 | (Pred0 == ICmpInst::ICMP_SGE && RangeStart->isZero()))) | ||||||||
| 705 | return nullptr; | ||||||||
| 706 | |||||||||
| 707 | ICmpInst::Predicate Pred1 = (Inverted ? Cmp1->getInversePredicate() : | ||||||||
| 708 | Cmp1->getPredicate()); | ||||||||
| 709 | |||||||||
| 710 | Value *Input = Cmp0->getOperand(0); | ||||||||
| 711 | Value *RangeEnd; | ||||||||
| 712 | if (Cmp1->getOperand(0) == Input) { | ||||||||
| 713 | // For the upper range compare we have: icmp x, n | ||||||||
| 714 | RangeEnd = Cmp1->getOperand(1); | ||||||||
| 715 | } else if (Cmp1->getOperand(1) == Input) { | ||||||||
| 716 | // For the upper range compare we have: icmp n, x | ||||||||
| 717 | RangeEnd = Cmp1->getOperand(0); | ||||||||
| 718 | Pred1 = ICmpInst::getSwappedPredicate(Pred1); | ||||||||
| 719 | } else { | ||||||||
| 720 | return nullptr; | ||||||||
| 721 | } | ||||||||
| 722 | |||||||||
| 723 | // Check the upper range comparison, e.g. x < n | ||||||||
| 724 | ICmpInst::Predicate NewPred; | ||||||||
| 725 | switch (Pred1) { | ||||||||
| 726 | case ICmpInst::ICMP_SLT: NewPred = ICmpInst::ICMP_ULT; break; | ||||||||
| 727 | case ICmpInst::ICMP_SLE: NewPred = ICmpInst::ICMP_ULE; break; | ||||||||
| 728 | default: return nullptr; | ||||||||
| 729 | } | ||||||||
| 730 | |||||||||
| 731 | // This simplification is only valid if the upper range is not negative. | ||||||||
| 732 | KnownBits Known = computeKnownBits(RangeEnd, /*Depth=*/0, Cmp1); | ||||||||
| 733 | if (!Known.isNonNegative()) | ||||||||
| 734 | return nullptr; | ||||||||
| 735 | |||||||||
| 736 | if (Inverted) | ||||||||
| 737 | NewPred = ICmpInst::getInversePredicate(NewPred); | ||||||||
| 738 | |||||||||
| 739 | return Builder.CreateICmp(NewPred, Input, RangeEnd); | ||||||||
| 740 | } | ||||||||
| 741 | |||||||||
| 742 | // Fold (iszero(A & K1) | iszero(A & K2)) -> (A & (K1 | K2)) != (K1 | K2) | ||||||||
| 743 | // Fold (!iszero(A & K1) & !iszero(A & K2)) -> (A & (K1 | K2)) == (K1 | K2) | ||||||||
| 744 | Value *InstCombinerImpl::foldAndOrOfICmpsOfAndWithPow2(ICmpInst *LHS, | ||||||||
| 745 | ICmpInst *RHS, | ||||||||
| 746 | Instruction *CxtI, | ||||||||
| 747 | bool IsAnd, | ||||||||
| 748 | bool IsLogical) { | ||||||||
| 749 | CmpInst::Predicate Pred = IsAnd ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ; | ||||||||
| 750 | if (LHS->getPredicate() != Pred || RHS->getPredicate() != Pred) | ||||||||
| 751 | return nullptr; | ||||||||
| 752 | |||||||||
| 753 | if (!match(LHS->getOperand(1), m_Zero()) || | ||||||||
| 754 | !match(RHS->getOperand(1), m_Zero())) | ||||||||
| 755 | return nullptr; | ||||||||
| 756 | |||||||||
| 757 | Value *L1, *L2, *R1, *R2; | ||||||||
| 758 | if (match(LHS->getOperand(0), m_And(m_Value(L1), m_Value(L2))) && | ||||||||
| 759 | match(RHS->getOperand(0), m_And(m_Value(R1), m_Value(R2)))) { | ||||||||
| 760 | if (L1 == R2 || L2 == R2) | ||||||||
| 761 | std::swap(R1, R2); | ||||||||
| 762 | if (L2 == R1) | ||||||||
| 763 | std::swap(L1, L2); | ||||||||
| 764 | |||||||||
| 765 | if (L1 == R1 && | ||||||||
| 766 | isKnownToBeAPowerOfTwo(L2, false, 0, CxtI) && | ||||||||
| 767 | isKnownToBeAPowerOfTwo(R2, false, 0, CxtI)) { | ||||||||
| 768 | // If this is a logical and/or, then we must prevent propagation of a | ||||||||
| 769 | // poison value from the RHS by inserting freeze. | ||||||||
| 770 | if (IsLogical) | ||||||||
| 771 | R2 = Builder.CreateFreeze(R2); | ||||||||
| 772 | Value *Mask = Builder.CreateOr(L2, R2); | ||||||||
| 773 | Value *Masked = Builder.CreateAnd(L1, Mask); | ||||||||
| 774 | auto NewPred = IsAnd ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE; | ||||||||
| 775 | return Builder.CreateICmp(NewPred, Masked, Mask); | ||||||||
| 776 | } | ||||||||
| 777 | } | ||||||||
| 778 | |||||||||
| 779 | return nullptr; | ||||||||
| 780 | } | ||||||||
| 781 | |||||||||
| 782 | /// General pattern: | ||||||||
| 783 | /// X & Y | ||||||||
| 784 | /// | ||||||||
| 785 | /// Where Y is checking that all the high bits (covered by a mask 4294967168) | ||||||||
| 786 | /// are uniform, i.e. %arg & 4294967168 can be either 4294967168 or 0 | ||||||||
| 787 | /// Pattern can be one of: | ||||||||
| 788 | /// %t = add i32 %arg, 128 | ||||||||
| 789 | /// %r = icmp ult i32 %t, 256 | ||||||||
| 790 | /// Or | ||||||||
| 791 | /// %t0 = shl i32 %arg, 24 | ||||||||
| 792 | /// %t1 = ashr i32 %t0, 24 | ||||||||
| 793 | /// %r = icmp eq i32 %t1, %arg | ||||||||
| 794 | /// Or | ||||||||
| 795 | /// %t0 = trunc i32 %arg to i8 | ||||||||
| 796 | /// %t1 = sext i8 %t0 to i32 | ||||||||
| 797 | /// %r = icmp eq i32 %t1, %arg | ||||||||
| 798 | /// This pattern is a signed truncation check. | ||||||||
| 799 | /// | ||||||||
| 800 | /// And X is checking that some bit in that same mask is zero. | ||||||||
| 801 | /// I.e. can be one of: | ||||||||
| 802 | /// %r = icmp sgt i32 %arg, -1 | ||||||||
| 803 | /// Or | ||||||||
| 804 | /// %t = and i32 %arg, 2147483648 | ||||||||
| 805 | /// %r = icmp eq i32 %t, 0 | ||||||||
| 806 | /// | ||||||||
| 807 | /// Since we are checking that all the bits in that mask are the same, | ||||||||
| 808 | /// and a particular bit is zero, what we are really checking is that all the | ||||||||
| 809 | /// masked bits are zero. | ||||||||
| 810 | /// So this should be transformed to: | ||||||||
| 811 | /// %r = icmp ult i32 %arg, 128 | ||||||||
| 812 | static Value *foldSignedTruncationCheck(ICmpInst *ICmp0, ICmpInst *ICmp1, | ||||||||
| 813 | Instruction &CxtI, | ||||||||
| 814 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 815 | assert(CxtI.getOpcode() == Instruction::And)(static_cast <bool> (CxtI.getOpcode() == Instruction::And ) ? void (0) : __assert_fail ("CxtI.getOpcode() == Instruction::And" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 815 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 816 | |||||||||
| 817 | // Match icmp ult (add %arg, C01), C1 (C1 == C01 << 1; powers of two) | ||||||||
| 818 | auto tryToMatchSignedTruncationCheck = [](ICmpInst *ICmp, Value *&X, | ||||||||
| 819 | APInt &SignBitMask) -> bool { | ||||||||
| 820 | CmpInst::Predicate Pred; | ||||||||
| 821 | const APInt *I01, *I1; // powers of two; I1 == I01 << 1 | ||||||||
| 822 | if (!(match(ICmp, | ||||||||
| 823 | m_ICmp(Pred, m_Add(m_Value(X), m_Power2(I01)), m_Power2(I1))) && | ||||||||
| 824 | Pred == ICmpInst::ICMP_ULT && I1->ugt(*I01) && I01->shl(1) == *I1)) | ||||||||
| 825 | return false; | ||||||||
| 826 | // Which bit is the new sign bit as per the 'signed truncation' pattern? | ||||||||
| 827 | SignBitMask = *I01; | ||||||||
| 828 | return true; | ||||||||
| 829 | }; | ||||||||
| 830 | |||||||||
| 831 | // One icmp needs to be 'signed truncation check'. | ||||||||
| 832 | // We need to match this first, else we will mismatch commutative cases. | ||||||||
| 833 | Value *X1; | ||||||||
| 834 | APInt HighestBit; | ||||||||
| 835 | ICmpInst *OtherICmp; | ||||||||
| 836 | if (tryToMatchSignedTruncationCheck(ICmp1, X1, HighestBit)) | ||||||||
| 837 | OtherICmp = ICmp0; | ||||||||
| 838 | else if (tryToMatchSignedTruncationCheck(ICmp0, X1, HighestBit)) | ||||||||
| 839 | OtherICmp = ICmp1; | ||||||||
| 840 | else | ||||||||
| 841 | return nullptr; | ||||||||
| 842 | |||||||||
| 843 | assert(HighestBit.isPowerOf2() && "expected to be power of two (non-zero)")(static_cast <bool> (HighestBit.isPowerOf2() && "expected to be power of two (non-zero)") ? void (0) : __assert_fail ("HighestBit.isPowerOf2() && \"expected to be power of two (non-zero)\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 843 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 844 | |||||||||
| 845 | // Try to match/decompose into: icmp eq (X & Mask), 0 | ||||||||
| 846 | auto tryToDecompose = [](ICmpInst *ICmp, Value *&X, | ||||||||
| 847 | APInt &UnsetBitsMask) -> bool { | ||||||||
| 848 | CmpInst::Predicate Pred = ICmp->getPredicate(); | ||||||||
| 849 | // Can it be decomposed into icmp eq (X & Mask), 0 ? | ||||||||
| 850 | if (llvm::decomposeBitTestICmp(ICmp->getOperand(0), ICmp->getOperand(1), | ||||||||
| 851 | Pred, X, UnsetBitsMask, | ||||||||
| 852 | /*LookThroughTrunc=*/false) && | ||||||||
| 853 | Pred == ICmpInst::ICMP_EQ) | ||||||||
| 854 | return true; | ||||||||
| 855 | // Is it icmp eq (X & Mask), 0 already? | ||||||||
| 856 | const APInt *Mask; | ||||||||
| 857 | if (match(ICmp, m_ICmp(Pred, m_And(m_Value(X), m_APInt(Mask)), m_Zero())) && | ||||||||
| 858 | Pred == ICmpInst::ICMP_EQ) { | ||||||||
| 859 | UnsetBitsMask = *Mask; | ||||||||
| 860 | return true; | ||||||||
| 861 | } | ||||||||
| 862 | return false; | ||||||||
| 863 | }; | ||||||||
| 864 | |||||||||
| 865 | // And the other icmp needs to be decomposable into a bit test. | ||||||||
| 866 | Value *X0; | ||||||||
| 867 | APInt UnsetBitsMask; | ||||||||
| 868 | if (!tryToDecompose(OtherICmp, X0, UnsetBitsMask)) | ||||||||
| 869 | return nullptr; | ||||||||
| 870 | |||||||||
| 871 | assert(!UnsetBitsMask.isZero() && "empty mask makes no sense.")(static_cast <bool> (!UnsetBitsMask.isZero() && "empty mask makes no sense.") ? void (0) : __assert_fail ("!UnsetBitsMask.isZero() && \"empty mask makes no sense.\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 871 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 872 | |||||||||
| 873 | // Are they working on the same value? | ||||||||
| 874 | Value *X; | ||||||||
| 875 | if (X1 == X0) { | ||||||||
| 876 | // Ok as is. | ||||||||
| 877 | X = X1; | ||||||||
| 878 | } else if (match(X0, m_Trunc(m_Specific(X1)))) { | ||||||||
| 879 | UnsetBitsMask = UnsetBitsMask.zext(X1->getType()->getScalarSizeInBits()); | ||||||||
| 880 | X = X1; | ||||||||
| 881 | } else | ||||||||
| 882 | return nullptr; | ||||||||
| 883 | |||||||||
| 884 | // So which bits should be uniform as per the 'signed truncation check'? | ||||||||
| 885 | // (all the bits starting with (i.e. including) HighestBit) | ||||||||
| 886 | APInt SignBitsMask = ~(HighestBit - 1U); | ||||||||
| 887 | |||||||||
| 888 | // UnsetBitsMask must have some common bits with SignBitsMask, | ||||||||
| 889 | if (!UnsetBitsMask.intersects(SignBitsMask)) | ||||||||
| 890 | return nullptr; | ||||||||
| 891 | |||||||||
| 892 | // Does UnsetBitsMask contain any bits outside of SignBitsMask? | ||||||||
| 893 | if (!UnsetBitsMask.isSubsetOf(SignBitsMask)) { | ||||||||
| 894 | APInt OtherHighestBit = (~UnsetBitsMask) + 1U; | ||||||||
| 895 | if (!OtherHighestBit.isPowerOf2()) | ||||||||
| 896 | return nullptr; | ||||||||
| 897 | HighestBit = APIntOps::umin(HighestBit, OtherHighestBit); | ||||||||
| 898 | } | ||||||||
| 899 | // Else, if it does not, then all is ok as-is. | ||||||||
| 900 | |||||||||
| 901 | // %r = icmp ult %X, SignBit | ||||||||
| 902 | return Builder.CreateICmpULT(X, ConstantInt::get(X->getType(), HighestBit), | ||||||||
| 903 | CxtI.getName() + ".simplified"); | ||||||||
| 904 | } | ||||||||
| 905 | |||||||||
| 906 | /// Fold (icmp eq ctpop(X) 1) | (icmp eq X 0) into (icmp ult ctpop(X) 2) and | ||||||||
| 907 | /// fold (icmp ne ctpop(X) 1) & (icmp ne X 0) into (icmp ugt ctpop(X) 1). | ||||||||
| 908 | /// Also used for logical and/or, must be poison safe. | ||||||||
| 909 | static Value *foldIsPowerOf2OrZero(ICmpInst *Cmp0, ICmpInst *Cmp1, bool IsAnd, | ||||||||
| 910 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 911 | CmpInst::Predicate Pred0, Pred1; | ||||||||
| 912 | Value *X; | ||||||||
| 913 | if (!match(Cmp0, m_ICmp(Pred0, m_Intrinsic<Intrinsic::ctpop>(m_Value(X)), | ||||||||
| 914 | m_SpecificInt(1))) || | ||||||||
| 915 | !match(Cmp1, m_ICmp(Pred1, m_Specific(X), m_ZeroInt()))) | ||||||||
| 916 | return nullptr; | ||||||||
| 917 | |||||||||
| 918 | Value *CtPop = Cmp0->getOperand(0); | ||||||||
| 919 | if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_NE) | ||||||||
| 920 | return Builder.CreateICmpUGT(CtPop, ConstantInt::get(CtPop->getType(), 1)); | ||||||||
| 921 | if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_EQ) | ||||||||
| 922 | return Builder.CreateICmpULT(CtPop, ConstantInt::get(CtPop->getType(), 2)); | ||||||||
| 923 | |||||||||
| 924 | return nullptr; | ||||||||
| 925 | } | ||||||||
| 926 | |||||||||
| 927 | /// Reduce a pair of compares that check if a value has exactly 1 bit set. | ||||||||
| 928 | /// Also used for logical and/or, must be poison safe. | ||||||||
| 929 | static Value *foldIsPowerOf2(ICmpInst *Cmp0, ICmpInst *Cmp1, bool JoinedByAnd, | ||||||||
| 930 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 931 | // Handle 'and' / 'or' commutation: make the equality check the first operand. | ||||||||
| 932 | if (JoinedByAnd && Cmp1->getPredicate() == ICmpInst::ICMP_NE) | ||||||||
| 933 | std::swap(Cmp0, Cmp1); | ||||||||
| 934 | else if (!JoinedByAnd && Cmp1->getPredicate() == ICmpInst::ICMP_EQ) | ||||||||
| 935 | std::swap(Cmp0, Cmp1); | ||||||||
| 936 | |||||||||
| 937 | // (X != 0) && (ctpop(X) u< 2) --> ctpop(X) == 1 | ||||||||
| 938 | CmpInst::Predicate Pred0, Pred1; | ||||||||
| 939 | Value *X; | ||||||||
| 940 | if (JoinedByAnd && match(Cmp0, m_ICmp(Pred0, m_Value(X), m_ZeroInt())) && | ||||||||
| 941 | match(Cmp1, m_ICmp(Pred1, m_Intrinsic<Intrinsic::ctpop>(m_Specific(X)), | ||||||||
| 942 | m_SpecificInt(2))) && | ||||||||
| 943 | Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_ULT) { | ||||||||
| 944 | Value *CtPop = Cmp1->getOperand(0); | ||||||||
| 945 | return Builder.CreateICmpEQ(CtPop, ConstantInt::get(CtPop->getType(), 1)); | ||||||||
| 946 | } | ||||||||
| 947 | // (X == 0) || (ctpop(X) u> 1) --> ctpop(X) != 1 | ||||||||
| 948 | if (!JoinedByAnd && match(Cmp0, m_ICmp(Pred0, m_Value(X), m_ZeroInt())) && | ||||||||
| 949 | match(Cmp1, m_ICmp(Pred1, m_Intrinsic<Intrinsic::ctpop>(m_Specific(X)), | ||||||||
| 950 | m_SpecificInt(1))) && | ||||||||
| 951 | Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_UGT) { | ||||||||
| 952 | Value *CtPop = Cmp1->getOperand(0); | ||||||||
| 953 | return Builder.CreateICmpNE(CtPop, ConstantInt::get(CtPop->getType(), 1)); | ||||||||
| 954 | } | ||||||||
| 955 | return nullptr; | ||||||||
| 956 | } | ||||||||
| 957 | |||||||||
| 958 | /// Commuted variants are assumed to be handled by calling this function again | ||||||||
| 959 | /// with the parameters swapped. | ||||||||
| 960 | static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp, | ||||||||
| 961 | ICmpInst *UnsignedICmp, bool IsAnd, | ||||||||
| 962 | const SimplifyQuery &Q, | ||||||||
| 963 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 964 | Value *ZeroCmpOp; | ||||||||
| 965 | ICmpInst::Predicate EqPred; | ||||||||
| 966 | if (!match(ZeroICmp, m_ICmp(EqPred, m_Value(ZeroCmpOp), m_Zero())) || | ||||||||
| 967 | !ICmpInst::isEquality(EqPred)) | ||||||||
| 968 | return nullptr; | ||||||||
| 969 | |||||||||
| 970 | auto IsKnownNonZero = [&](Value *V) { | ||||||||
| 971 | return isKnownNonZero(V, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT); | ||||||||
| 972 | }; | ||||||||
| 973 | |||||||||
| 974 | ICmpInst::Predicate UnsignedPred; | ||||||||
| 975 | |||||||||
| 976 | Value *A, *B; | ||||||||
| 977 | if (match(UnsignedICmp, | ||||||||
| 978 | m_c_ICmp(UnsignedPred, m_Specific(ZeroCmpOp), m_Value(A))) && | ||||||||
| 979 | match(ZeroCmpOp, m_c_Add(m_Specific(A), m_Value(B))) && | ||||||||
| 980 | (ZeroICmp->hasOneUse() || UnsignedICmp->hasOneUse())) { | ||||||||
| 981 | auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) { | ||||||||
| 982 | if (!IsKnownNonZero(NonZero)) | ||||||||
| 983 | std::swap(NonZero, Other); | ||||||||
| 984 | return IsKnownNonZero(NonZero); | ||||||||
| 985 | }; | ||||||||
| 986 | |||||||||
| 987 | // Given ZeroCmpOp = (A + B) | ||||||||
| 988 | // ZeroCmpOp < A && ZeroCmpOp != 0 --> (0-X) < Y iff | ||||||||
| 989 | // ZeroCmpOp >= A || ZeroCmpOp == 0 --> (0-X) >= Y iff | ||||||||
| 990 | // with X being the value (A/B) that is known to be non-zero, | ||||||||
| 991 | // and Y being remaining value. | ||||||||
| 992 | if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_NE && | ||||||||
| 993 | IsAnd && GetKnownNonZeroAndOther(B, A)) | ||||||||
| 994 | return Builder.CreateICmpULT(Builder.CreateNeg(B), A); | ||||||||
| 995 | if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ && | ||||||||
| 996 | !IsAnd && GetKnownNonZeroAndOther(B, A)) | ||||||||
| 997 | return Builder.CreateICmpUGE(Builder.CreateNeg(B), A); | ||||||||
| 998 | } | ||||||||
| 999 | |||||||||
| 1000 | Value *Base, *Offset; | ||||||||
| 1001 | if (!match(ZeroCmpOp, m_Sub(m_Value(Base), m_Value(Offset)))) | ||||||||
| 1002 | return nullptr; | ||||||||
| 1003 | |||||||||
| 1004 | if (!match(UnsignedICmp, | ||||||||
| 1005 | m_c_ICmp(UnsignedPred, m_Specific(Base), m_Specific(Offset))) || | ||||||||
| 1006 | !ICmpInst::isUnsigned(UnsignedPred)) | ||||||||
| 1007 | return nullptr; | ||||||||
| 1008 | |||||||||
| 1009 | // Base >=/> Offset && (Base - Offset) != 0 <--> Base > Offset | ||||||||
| 1010 | // (no overflow and not null) | ||||||||
| 1011 | if ((UnsignedPred == ICmpInst::ICMP_UGE || | ||||||||
| 1012 | UnsignedPred == ICmpInst::ICMP_UGT) && | ||||||||
| 1013 | EqPred == ICmpInst::ICMP_NE && IsAnd) | ||||||||
| 1014 | return Builder.CreateICmpUGT(Base, Offset); | ||||||||
| 1015 | |||||||||
| 1016 | // Base <=/< Offset || (Base - Offset) == 0 <--> Base <= Offset | ||||||||
| 1017 | // (overflow or null) | ||||||||
| 1018 | if ((UnsignedPred == ICmpInst::ICMP_ULE || | ||||||||
| 1019 | UnsignedPred == ICmpInst::ICMP_ULT) && | ||||||||
| 1020 | EqPred == ICmpInst::ICMP_EQ && !IsAnd) | ||||||||
| 1021 | return Builder.CreateICmpULE(Base, Offset); | ||||||||
| 1022 | |||||||||
| 1023 | // Base <= Offset && (Base - Offset) != 0 --> Base < Offset | ||||||||
| 1024 | if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE && | ||||||||
| 1025 | IsAnd) | ||||||||
| 1026 | return Builder.CreateICmpULT(Base, Offset); | ||||||||
| 1027 | |||||||||
| 1028 | // Base > Offset || (Base - Offset) == 0 --> Base >= Offset | ||||||||
| 1029 | if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ && | ||||||||
| 1030 | !IsAnd) | ||||||||
| 1031 | return Builder.CreateICmpUGE(Base, Offset); | ||||||||
| 1032 | |||||||||
| 1033 | return nullptr; | ||||||||
| 1034 | } | ||||||||
| 1035 | |||||||||
| 1036 | struct IntPart { | ||||||||
| 1037 | Value *From; | ||||||||
| 1038 | unsigned StartBit; | ||||||||
| 1039 | unsigned NumBits; | ||||||||
| 1040 | }; | ||||||||
| 1041 | |||||||||
| 1042 | /// Match an extraction of bits from an integer. | ||||||||
| 1043 | static std::optional<IntPart> matchIntPart(Value *V) { | ||||||||
| 1044 | Value *X; | ||||||||
| 1045 | if (!match(V, m_OneUse(m_Trunc(m_Value(X))))) | ||||||||
| 1046 | return std::nullopt; | ||||||||
| 1047 | |||||||||
| 1048 | unsigned NumOriginalBits = X->getType()->getScalarSizeInBits(); | ||||||||
| 1049 | unsigned NumExtractedBits = V->getType()->getScalarSizeInBits(); | ||||||||
| 1050 | Value *Y; | ||||||||
| 1051 | const APInt *Shift; | ||||||||
| 1052 | // For a trunc(lshr Y, Shift) pattern, make sure we're only extracting bits | ||||||||
| 1053 | // from Y, not any shifted-in zeroes. | ||||||||
| 1054 | if (match(X, m_OneUse(m_LShr(m_Value(Y), m_APInt(Shift)))) && | ||||||||
| 1055 | Shift->ule(NumOriginalBits - NumExtractedBits)) | ||||||||
| 1056 | return {{Y, (unsigned)Shift->getZExtValue(), NumExtractedBits}}; | ||||||||
| 1057 | return {{X, 0, NumExtractedBits}}; | ||||||||
| 1058 | } | ||||||||
| 1059 | |||||||||
| 1060 | /// Materialize an extraction of bits from an integer in IR. | ||||||||
| 1061 | static Value *extractIntPart(const IntPart &P, IRBuilderBase &Builder) { | ||||||||
| 1062 | Value *V = P.From; | ||||||||
| 1063 | if (P.StartBit) | ||||||||
| 1064 | V = Builder.CreateLShr(V, P.StartBit); | ||||||||
| 1065 | Type *TruncTy = V->getType()->getWithNewBitWidth(P.NumBits); | ||||||||
| 1066 | if (TruncTy != V->getType()) | ||||||||
| 1067 | V = Builder.CreateTrunc(V, TruncTy); | ||||||||
| 1068 | return V; | ||||||||
| 1069 | } | ||||||||
| 1070 | |||||||||
| 1071 | /// (icmp eq X0, Y0) & (icmp eq X1, Y1) -> icmp eq X01, Y01 | ||||||||
| 1072 | /// (icmp ne X0, Y0) | (icmp ne X1, Y1) -> icmp ne X01, Y01 | ||||||||
| 1073 | /// where X0, X1 and Y0, Y1 are adjacent parts extracted from an integer. | ||||||||
| 1074 | Value *InstCombinerImpl::foldEqOfParts(ICmpInst *Cmp0, ICmpInst *Cmp1, | ||||||||
| 1075 | bool IsAnd) { | ||||||||
| 1076 | if (!Cmp0->hasOneUse() || !Cmp1->hasOneUse()) | ||||||||
| 1077 | return nullptr; | ||||||||
| 1078 | |||||||||
| 1079 | CmpInst::Predicate Pred = IsAnd ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE; | ||||||||
| 1080 | if (Cmp0->getPredicate() != Pred || Cmp1->getPredicate() != Pred) | ||||||||
| 1081 | return nullptr; | ||||||||
| 1082 | |||||||||
| 1083 | std::optional<IntPart> L0 = matchIntPart(Cmp0->getOperand(0)); | ||||||||
| 1084 | std::optional<IntPart> R0 = matchIntPart(Cmp0->getOperand(1)); | ||||||||
| 1085 | std::optional<IntPart> L1 = matchIntPart(Cmp1->getOperand(0)); | ||||||||
| 1086 | std::optional<IntPart> R1 = matchIntPart(Cmp1->getOperand(1)); | ||||||||
| 1087 | if (!L0 || !R0 || !L1 || !R1) | ||||||||
| 1088 | return nullptr; | ||||||||
| 1089 | |||||||||
| 1090 | // Make sure the LHS/RHS compare a part of the same value, possibly after | ||||||||
| 1091 | // an operand swap. | ||||||||
| 1092 | if (L0->From != L1->From || R0->From != R1->From) { | ||||||||
| 1093 | if (L0->From != R1->From || R0->From != L1->From) | ||||||||
| 1094 | return nullptr; | ||||||||
| 1095 | std::swap(L1, R1); | ||||||||
| 1096 | } | ||||||||
| 1097 | |||||||||
| 1098 | // Make sure the extracted parts are adjacent, canonicalizing to L0/R0 being | ||||||||
| 1099 | // the low part and L1/R1 being the high part. | ||||||||
| 1100 | if (L0->StartBit + L0->NumBits != L1->StartBit || | ||||||||
| 1101 | R0->StartBit + R0->NumBits != R1->StartBit) { | ||||||||
| 1102 | if (L1->StartBit + L1->NumBits != L0->StartBit || | ||||||||
| 1103 | R1->StartBit + R1->NumBits != R0->StartBit) | ||||||||
| 1104 | return nullptr; | ||||||||
| 1105 | std::swap(L0, L1); | ||||||||
| 1106 | std::swap(R0, R1); | ||||||||
| 1107 | } | ||||||||
| 1108 | |||||||||
| 1109 | // We can simplify to a comparison of these larger parts of the integers. | ||||||||
| 1110 | IntPart L = {L0->From, L0->StartBit, L0->NumBits + L1->NumBits}; | ||||||||
| 1111 | IntPart R = {R0->From, R0->StartBit, R0->NumBits + R1->NumBits}; | ||||||||
| 1112 | Value *LValue = extractIntPart(L, Builder); | ||||||||
| 1113 | Value *RValue = extractIntPart(R, Builder); | ||||||||
| 1114 | return Builder.CreateICmp(Pred, LValue, RValue); | ||||||||
| 1115 | } | ||||||||
| 1116 | |||||||||
| 1117 | /// Reduce logic-of-compares with equality to a constant by substituting a | ||||||||
| 1118 | /// common operand with the constant. Callers are expected to call this with | ||||||||
| 1119 | /// Cmp0/Cmp1 switched to handle logic op commutativity. | ||||||||
| 1120 | static Value *foldAndOrOfICmpsWithConstEq(ICmpInst *Cmp0, ICmpInst *Cmp1, | ||||||||
| 1121 | bool IsAnd, bool IsLogical, | ||||||||
| 1122 | InstCombiner::BuilderTy &Builder, | ||||||||
| 1123 | const SimplifyQuery &Q) { | ||||||||
| 1124 | // Match an equality compare with a non-poison constant as Cmp0. | ||||||||
| 1125 | // Also, give up if the compare can be constant-folded to avoid looping. | ||||||||
| 1126 | ICmpInst::Predicate Pred0; | ||||||||
| 1127 | Value *X; | ||||||||
| 1128 | Constant *C; | ||||||||
| 1129 | if (!match(Cmp0, m_ICmp(Pred0, m_Value(X), m_Constant(C))) || | ||||||||
| 1130 | !isGuaranteedNotToBeUndefOrPoison(C) || isa<Constant>(X)) | ||||||||
| 1131 | return nullptr; | ||||||||
| 1132 | if ((IsAnd && Pred0 != ICmpInst::ICMP_EQ) || | ||||||||
| 1133 | (!IsAnd && Pred0 != ICmpInst::ICMP_NE)) | ||||||||
| 1134 | return nullptr; | ||||||||
| 1135 | |||||||||
| 1136 | // The other compare must include a common operand (X). Canonicalize the | ||||||||
| 1137 | // common operand as operand 1 (Pred1 is swapped if the common operand was | ||||||||
| 1138 | // operand 0). | ||||||||
| 1139 | Value *Y; | ||||||||
| 1140 | ICmpInst::Predicate Pred1; | ||||||||
| 1141 | if (!match(Cmp1, m_c_ICmp(Pred1, m_Value(Y), m_Deferred(X)))) | ||||||||
| 1142 | return nullptr; | ||||||||
| 1143 | |||||||||
| 1144 | // Replace variable with constant value equivalence to remove a variable use: | ||||||||
| 1145 | // (X == C) && (Y Pred1 X) --> (X == C) && (Y Pred1 C) | ||||||||
| 1146 | // (X != C) || (Y Pred1 X) --> (X != C) || (Y Pred1 C) | ||||||||
| 1147 | // Can think of the 'or' substitution with the 'and' bool equivalent: | ||||||||
| 1148 | // A || B --> A || (!A && B) | ||||||||
| 1149 | Value *SubstituteCmp = simplifyICmpInst(Pred1, Y, C, Q); | ||||||||
| 1150 | if (!SubstituteCmp) { | ||||||||
| 1151 | // If we need to create a new instruction, require that the old compare can | ||||||||
| 1152 | // be removed. | ||||||||
| 1153 | if (!Cmp1->hasOneUse()) | ||||||||
| 1154 | return nullptr; | ||||||||
| 1155 | SubstituteCmp = Builder.CreateICmp(Pred1, Y, C); | ||||||||
| 1156 | } | ||||||||
| 1157 | if (IsLogical) | ||||||||
| 1158 | return IsAnd ? Builder.CreateLogicalAnd(Cmp0, SubstituteCmp) | ||||||||
| 1159 | : Builder.CreateLogicalOr(Cmp0, SubstituteCmp); | ||||||||
| 1160 | return Builder.CreateBinOp(IsAnd ? Instruction::And : Instruction::Or, Cmp0, | ||||||||
| 1161 | SubstituteCmp); | ||||||||
| 1162 | } | ||||||||
| 1163 | |||||||||
| 1164 | /// Fold (icmp Pred1 V1, C1) & (icmp Pred2 V2, C2) | ||||||||
| 1165 | /// or (icmp Pred1 V1, C1) | (icmp Pred2 V2, C2) | ||||||||
| 1166 | /// into a single comparison using range-based reasoning. | ||||||||
| 1167 | /// NOTE: This is also used for logical and/or, must be poison-safe! | ||||||||
| 1168 | Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(ICmpInst *ICmp1, | ||||||||
| 1169 | ICmpInst *ICmp2, | ||||||||
| 1170 | bool IsAnd) { | ||||||||
| 1171 | ICmpInst::Predicate Pred1, Pred2; | ||||||||
| 1172 | Value *V1, *V2; | ||||||||
| 1173 | const APInt *C1, *C2; | ||||||||
| 1174 | if (!match(ICmp1, m_ICmp(Pred1, m_Value(V1), m_APInt(C1))) || | ||||||||
| 1175 | !match(ICmp2, m_ICmp(Pred2, m_Value(V2), m_APInt(C2)))) | ||||||||
| 1176 | return nullptr; | ||||||||
| 1177 | |||||||||
| 1178 | // Look through add of a constant offset on V1, V2, or both operands. This | ||||||||
| 1179 | // allows us to interpret the V + C' < C'' range idiom into a proper range. | ||||||||
| 1180 | const APInt *Offset1 = nullptr, *Offset2 = nullptr; | ||||||||
| 1181 | if (V1 != V2) { | ||||||||
| 1182 | Value *X; | ||||||||
| 1183 | if (match(V1, m_Add(m_Value(X), m_APInt(Offset1)))) | ||||||||
| 1184 | V1 = X; | ||||||||
| 1185 | if (match(V2, m_Add(m_Value(X), m_APInt(Offset2)))) | ||||||||
| 1186 | V2 = X; | ||||||||
| 1187 | } | ||||||||
| 1188 | |||||||||
| 1189 | if (V1 != V2) | ||||||||
| 1190 | return nullptr; | ||||||||
| 1191 | |||||||||
| 1192 | ConstantRange CR1 = ConstantRange::makeExactICmpRegion( | ||||||||
| 1193 | IsAnd ? ICmpInst::getInversePredicate(Pred1) : Pred1, *C1); | ||||||||
| 1194 | if (Offset1) | ||||||||
| 1195 | CR1 = CR1.subtract(*Offset1); | ||||||||
| 1196 | |||||||||
| 1197 | ConstantRange CR2 = ConstantRange::makeExactICmpRegion( | ||||||||
| 1198 | IsAnd ? ICmpInst::getInversePredicate(Pred2) : Pred2, *C2); | ||||||||
| 1199 | if (Offset2) | ||||||||
| 1200 | CR2 = CR2.subtract(*Offset2); | ||||||||
| 1201 | |||||||||
| 1202 | Type *Ty = V1->getType(); | ||||||||
| 1203 | Value *NewV = V1; | ||||||||
| 1204 | std::optional<ConstantRange> CR = CR1.exactUnionWith(CR2); | ||||||||
| 1205 | if (!CR) { | ||||||||
| 1206 | if (!(ICmp1->hasOneUse() && ICmp2->hasOneUse()) || CR1.isWrappedSet() || | ||||||||
| 1207 | CR2.isWrappedSet()) | ||||||||
| 1208 | return nullptr; | ||||||||
| 1209 | |||||||||
| 1210 | // Check whether we have equal-size ranges that only differ by one bit. | ||||||||
| 1211 | // In that case we can apply a mask to map one range onto the other. | ||||||||
| 1212 | APInt LowerDiff = CR1.getLower() ^ CR2.getLower(); | ||||||||
| 1213 | APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1); | ||||||||
| 1214 | APInt CR1Size = CR1.getUpper() - CR1.getLower(); | ||||||||
| 1215 | if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff || | ||||||||
| 1216 | CR1Size != CR2.getUpper() - CR2.getLower()) | ||||||||
| 1217 | return nullptr; | ||||||||
| 1218 | |||||||||
| 1219 | CR = CR1.getLower().ult(CR2.getLower()) ? CR1 : CR2; | ||||||||
| 1220 | NewV = Builder.CreateAnd(NewV, ConstantInt::get(Ty, ~LowerDiff)); | ||||||||
| 1221 | } | ||||||||
| 1222 | |||||||||
| 1223 | if (IsAnd) | ||||||||
| 1224 | CR = CR->inverse(); | ||||||||
| 1225 | |||||||||
| 1226 | CmpInst::Predicate NewPred; | ||||||||
| 1227 | APInt NewC, Offset; | ||||||||
| 1228 | CR->getEquivalentICmp(NewPred, NewC, Offset); | ||||||||
| 1229 | |||||||||
| 1230 | if (Offset != 0) | ||||||||
| 1231 | NewV = Builder.CreateAdd(NewV, ConstantInt::get(Ty, Offset)); | ||||||||
| 1232 | return Builder.CreateICmp(NewPred, NewV, ConstantInt::get(Ty, NewC)); | ||||||||
| 1233 | } | ||||||||
| 1234 | |||||||||
| 1235 | /// Ignore all operations which only change the sign of a value, returning the | ||||||||
| 1236 | /// underlying magnitude value. | ||||||||
| 1237 | static Value *stripSignOnlyFPOps(Value *Val) { | ||||||||
| 1238 | match(Val, m_FNeg(m_Value(Val))); | ||||||||
| 1239 | match(Val, m_FAbs(m_Value(Val))); | ||||||||
| 1240 | match(Val, m_CopySign(m_Value(Val), m_Value())); | ||||||||
| 1241 | return Val; | ||||||||
| 1242 | } | ||||||||
| 1243 | |||||||||
| 1244 | /// Matches canonical form of isnan, fcmp ord x, 0 | ||||||||
| 1245 | static bool matchIsNotNaN(FCmpInst::Predicate P, Value *LHS, Value *RHS) { | ||||||||
| 1246 | return P == FCmpInst::FCMP_ORD && match(RHS, m_AnyZeroFP()); | ||||||||
| 1247 | } | ||||||||
| 1248 | |||||||||
| 1249 | /// Matches fcmp u__ x, +/-inf | ||||||||
| 1250 | static bool matchUnorderedInfCompare(FCmpInst::Predicate P, Value *LHS, | ||||||||
| 1251 | Value *RHS) { | ||||||||
| 1252 | return FCmpInst::isUnordered(P) && match(RHS, m_Inf()); | ||||||||
| 1253 | } | ||||||||
| 1254 | |||||||||
| 1255 | /// and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf | ||||||||
| 1256 | /// | ||||||||
| 1257 | /// Clang emits this pattern for doing an isfinite check in __builtin_isnormal. | ||||||||
| 1258 | static Value *matchIsFiniteTest(InstCombiner::BuilderTy &Builder, FCmpInst *LHS, | ||||||||
| 1259 | FCmpInst *RHS) { | ||||||||
| 1260 | Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1); | ||||||||
| 1261 | Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1); | ||||||||
| 1262 | FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate(); | ||||||||
| 1263 | |||||||||
| 1264 | if (!matchIsNotNaN(PredL, LHS0, LHS1) || | ||||||||
| 1265 | !matchUnorderedInfCompare(PredR, RHS0, RHS1)) | ||||||||
| 1266 | return nullptr; | ||||||||
| 1267 | |||||||||
| 1268 | IRBuilder<>::FastMathFlagGuard FMFG(Builder); | ||||||||
| 1269 | FastMathFlags FMF = LHS->getFastMathFlags(); | ||||||||
| 1270 | FMF &= RHS->getFastMathFlags(); | ||||||||
| 1271 | Builder.setFastMathFlags(FMF); | ||||||||
| 1272 | |||||||||
| 1273 | return Builder.CreateFCmp(FCmpInst::getOrderedPredicate(PredR), RHS0, RHS1); | ||||||||
| 1274 | } | ||||||||
| 1275 | |||||||||
| 1276 | Value *InstCombinerImpl::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS, | ||||||||
| 1277 | bool IsAnd, bool IsLogicalSelect) { | ||||||||
| 1278 | Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1); | ||||||||
| 1279 | Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1); | ||||||||
| 1280 | FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate(); | ||||||||
| 1281 | |||||||||
| 1282 | if (LHS0 == RHS1 && RHS0 == LHS1) { | ||||||||
| 1283 | // Swap RHS operands to match LHS. | ||||||||
| 1284 | PredR = FCmpInst::getSwappedPredicate(PredR); | ||||||||
| 1285 | std::swap(RHS0, RHS1); | ||||||||
| 1286 | } | ||||||||
| 1287 | |||||||||
| 1288 | // Simplify (fcmp cc0 x, y) & (fcmp cc1 x, y). | ||||||||
| 1289 | // Suppose the relation between x and y is R, where R is one of | ||||||||
| 1290 | // U(1000), L(0100), G(0010) or E(0001), and CC0 and CC1 are the bitmasks for | ||||||||
| 1291 | // testing the desired relations. | ||||||||
| 1292 | // | ||||||||
| 1293 | // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this: | ||||||||
| 1294 | // bool(R & CC0) && bool(R & CC1) | ||||||||
| 1295 | // = bool((R & CC0) & (R & CC1)) | ||||||||
| 1296 | // = bool(R & (CC0 & CC1)) <= by re-association, commutation, and idempotency | ||||||||
| 1297 | // | ||||||||
| 1298 | // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this: | ||||||||
| 1299 | // bool(R & CC0) || bool(R & CC1) | ||||||||
| 1300 | // = bool((R & CC0) | (R & CC1)) | ||||||||
| 1301 | // = bool(R & (CC0 | CC1)) <= by reversed distribution (contribution? ;) | ||||||||
| 1302 | if (LHS0 == RHS0 && LHS1 == RHS1) { | ||||||||
| 1303 | unsigned FCmpCodeL = getFCmpCode(PredL); | ||||||||
| 1304 | unsigned FCmpCodeR = getFCmpCode(PredR); | ||||||||
| 1305 | unsigned NewPred = IsAnd ? FCmpCodeL & FCmpCodeR : FCmpCodeL | FCmpCodeR; | ||||||||
| 1306 | |||||||||
| 1307 | // Intersect the fast math flags. | ||||||||
| 1308 | // TODO: We can union the fast math flags unless this is a logical select. | ||||||||
| 1309 | IRBuilder<>::FastMathFlagGuard FMFG(Builder); | ||||||||
| 1310 | FastMathFlags FMF = LHS->getFastMathFlags(); | ||||||||
| 1311 | FMF &= RHS->getFastMathFlags(); | ||||||||
| 1312 | Builder.setFastMathFlags(FMF); | ||||||||
| 1313 | |||||||||
| 1314 | return getFCmpValue(NewPred, LHS0, LHS1, Builder); | ||||||||
| 1315 | } | ||||||||
| 1316 | |||||||||
| 1317 | // This transform is not valid for a logical select. | ||||||||
| 1318 | if (!IsLogicalSelect && | ||||||||
| 1319 | ((PredL == FCmpInst::FCMP_ORD && PredR == FCmpInst::FCMP_ORD && IsAnd) || | ||||||||
| 1320 | (PredL == FCmpInst::FCMP_UNO && PredR == FCmpInst::FCMP_UNO && | ||||||||
| 1321 | !IsAnd))) { | ||||||||
| 1322 | if (LHS0->getType() != RHS0->getType()) | ||||||||
| 1323 | return nullptr; | ||||||||
| 1324 | |||||||||
| 1325 | // FCmp canonicalization ensures that (fcmp ord/uno X, X) and | ||||||||
| 1326 | // (fcmp ord/uno X, C) will be transformed to (fcmp X, +0.0). | ||||||||
| 1327 | if (match(LHS1, m_PosZeroFP()) && match(RHS1, m_PosZeroFP())) | ||||||||
| 1328 | // Ignore the constants because they are obviously not NANs: | ||||||||
| 1329 | // (fcmp ord x, 0.0) & (fcmp ord y, 0.0) -> (fcmp ord x, y) | ||||||||
| 1330 | // (fcmp uno x, 0.0) | (fcmp uno y, 0.0) -> (fcmp uno x, y) | ||||||||
| 1331 | return Builder.CreateFCmp(PredL, LHS0, RHS0); | ||||||||
| 1332 | } | ||||||||
| 1333 | |||||||||
| 1334 | if (IsAnd && stripSignOnlyFPOps(LHS0) == stripSignOnlyFPOps(RHS0)) { | ||||||||
| 1335 | // and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf | ||||||||
| 1336 | // and (fcmp ord x, 0), (fcmp u* fabs(x), inf) -> fcmp o* x, inf | ||||||||
| 1337 | if (Value *Left = matchIsFiniteTest(Builder, LHS, RHS)) | ||||||||
| 1338 | return Left; | ||||||||
| 1339 | if (Value *Right = matchIsFiniteTest(Builder, RHS, LHS)) | ||||||||
| 1340 | return Right; | ||||||||
| 1341 | } | ||||||||
| 1342 | |||||||||
| 1343 | // Turn at least two fcmps with constants into llvm.is.fpclass. | ||||||||
| 1344 | // | ||||||||
| 1345 | // If we can represent a combined value test with one class call, we can | ||||||||
| 1346 | // potentially eliminate 4-6 instructions. If we can represent a test with a | ||||||||
| 1347 | // single fcmp with fneg and fabs, that's likely a better canonical form. | ||||||||
| 1348 | if (LHS->hasOneUse() && RHS->hasOneUse()) { | ||||||||
| 1349 | auto [ClassValRHS, ClassMaskRHS] = | ||||||||
| 1350 | fcmpToClassTest(PredR, *RHS->getFunction(), RHS0, RHS1); | ||||||||
| 1351 | if (ClassValRHS) { | ||||||||
| 1352 | auto [ClassValLHS, ClassMaskLHS] = | ||||||||
| 1353 | fcmpToClassTest(PredL, *LHS->getFunction(), LHS0, LHS1); | ||||||||
| 1354 | if (ClassValLHS == ClassValRHS) { | ||||||||
| 1355 | unsigned CombinedMask = IsAnd ? (ClassMaskLHS & ClassMaskRHS) | ||||||||
| 1356 | : (ClassMaskLHS | ClassMaskRHS); | ||||||||
| 1357 | return Builder.CreateIntrinsic( | ||||||||
| 1358 | Intrinsic::is_fpclass, {ClassValLHS->getType()}, | ||||||||
| 1359 | {ClassValLHS, Builder.getInt32(CombinedMask)}); | ||||||||
| 1360 | } | ||||||||
| 1361 | } | ||||||||
| 1362 | } | ||||||||
| 1363 | |||||||||
| 1364 | return nullptr; | ||||||||
| 1365 | } | ||||||||
| 1366 | |||||||||
| 1367 | /// Match an fcmp against a special value that performs a test possible by | ||||||||
| 1368 | /// llvm.is.fpclass. | ||||||||
| 1369 | static bool matchIsFPClassLikeFCmp(Value *Op, Value *&ClassVal, | ||||||||
| 1370 | uint64_t &ClassMask) { | ||||||||
| 1371 | auto *FCmp = dyn_cast<FCmpInst>(Op); | ||||||||
| 1372 | if (!FCmp || !FCmp->hasOneUse()) | ||||||||
| 1373 | return false; | ||||||||
| 1374 | |||||||||
| 1375 | std::tie(ClassVal, ClassMask) = | ||||||||
| 1376 | fcmpToClassTest(FCmp->getPredicate(), *FCmp->getParent()->getParent(), | ||||||||
| 1377 | FCmp->getOperand(0), FCmp->getOperand(1)); | ||||||||
| 1378 | return ClassVal != nullptr; | ||||||||
| 1379 | } | ||||||||
| 1380 | |||||||||
| 1381 | /// or (is_fpclass x, mask0), (is_fpclass x, mask1) | ||||||||
| 1382 | /// -> is_fpclass x, (mask0 | mask1) | ||||||||
| 1383 | /// and (is_fpclass x, mask0), (is_fpclass x, mask1) | ||||||||
| 1384 | /// -> is_fpclass x, (mask0 & mask1) | ||||||||
| 1385 | /// xor (is_fpclass x, mask0), (is_fpclass x, mask1) | ||||||||
| 1386 | /// -> is_fpclass x, (mask0 ^ mask1) | ||||||||
| 1387 | Instruction *InstCombinerImpl::foldLogicOfIsFPClass(BinaryOperator &BO, | ||||||||
| 1388 | Value *Op0, Value *Op1) { | ||||||||
| 1389 | Value *ClassVal0 = nullptr; | ||||||||
| 1390 | Value *ClassVal1 = nullptr; | ||||||||
| 1391 | uint64_t ClassMask0, ClassMask1; | ||||||||
| 1392 | |||||||||
| 1393 | // Restrict to folding one fcmp into one is.fpclass for now, don't introduce a | ||||||||
| 1394 | // new class. | ||||||||
| 1395 | // | ||||||||
| 1396 | // TODO: Support forming is.fpclass out of 2 separate fcmps when codegen is | ||||||||
| 1397 | // better. | ||||||||
| 1398 | |||||||||
| 1399 | bool IsLHSClass = | ||||||||
| 1400 | match(Op0, m_OneUse(m_Intrinsic<Intrinsic::is_fpclass>( | ||||||||
| 1401 | m_Value(ClassVal0), m_ConstantInt(ClassMask0)))); | ||||||||
| 1402 | bool IsRHSClass = | ||||||||
| 1403 | match(Op1, m_OneUse(m_Intrinsic<Intrinsic::is_fpclass>( | ||||||||
| 1404 | m_Value(ClassVal1), m_ConstantInt(ClassMask1)))); | ||||||||
| 1405 | if ((((IsLHSClass || matchIsFPClassLikeFCmp(Op0, ClassVal0, ClassMask0)) && | ||||||||
| 1406 | (IsRHSClass || matchIsFPClassLikeFCmp(Op1, ClassVal1, ClassMask1)))) && | ||||||||
| 1407 | ClassVal0 == ClassVal1) { | ||||||||
| 1408 | unsigned NewClassMask; | ||||||||
| 1409 | switch (BO.getOpcode()) { | ||||||||
| 1410 | case Instruction::And: | ||||||||
| 1411 | NewClassMask = ClassMask0 & ClassMask1; | ||||||||
| 1412 | break; | ||||||||
| 1413 | case Instruction::Or: | ||||||||
| 1414 | NewClassMask = ClassMask0 | ClassMask1; | ||||||||
| 1415 | break; | ||||||||
| 1416 | case Instruction::Xor: | ||||||||
| 1417 | NewClassMask = ClassMask0 ^ ClassMask1; | ||||||||
| 1418 | break; | ||||||||
| 1419 | default: | ||||||||
| 1420 | llvm_unreachable("not a binary logic operator")::llvm::llvm_unreachable_internal("not a binary logic operator" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1420 ); | ||||||||
| 1421 | } | ||||||||
| 1422 | |||||||||
| 1423 | if (IsLHSClass) { | ||||||||
| 1424 | auto *II = cast<IntrinsicInst>(Op0); | ||||||||
| 1425 | II->setArgOperand( | ||||||||
| 1426 | 1, ConstantInt::get(II->getArgOperand(1)->getType(), NewClassMask)); | ||||||||
| 1427 | return replaceInstUsesWith(BO, II); | ||||||||
| 1428 | } | ||||||||
| 1429 | |||||||||
| 1430 | if (IsRHSClass) { | ||||||||
| 1431 | auto *II = cast<IntrinsicInst>(Op1); | ||||||||
| 1432 | II->setArgOperand( | ||||||||
| 1433 | 1, ConstantInt::get(II->getArgOperand(1)->getType(), NewClassMask)); | ||||||||
| 1434 | return replaceInstUsesWith(BO, II); | ||||||||
| 1435 | } | ||||||||
| 1436 | |||||||||
| 1437 | CallInst *NewClass = | ||||||||
| 1438 | Builder.CreateIntrinsic(Intrinsic::is_fpclass, {ClassVal0->getType()}, | ||||||||
| 1439 | {ClassVal0, Builder.getInt32(NewClassMask)}); | ||||||||
| 1440 | return replaceInstUsesWith(BO, NewClass); | ||||||||
| 1441 | } | ||||||||
| 1442 | |||||||||
| 1443 | return nullptr; | ||||||||
| 1444 | } | ||||||||
| 1445 | |||||||||
| 1446 | /// Look for the pattern that conditionally negates a value via math operations: | ||||||||
| 1447 | /// cond.splat = sext i1 cond | ||||||||
| 1448 | /// sub = add cond.splat, x | ||||||||
| 1449 | /// xor = xor sub, cond.splat | ||||||||
| 1450 | /// and rewrite it to do the same, but via logical operations: | ||||||||
| 1451 | /// value.neg = sub 0, value | ||||||||
| 1452 | /// cond = select i1 neg, value.neg, value | ||||||||
| 1453 | Instruction *InstCombinerImpl::canonicalizeConditionalNegationViaMathToSelect( | ||||||||
| 1454 | BinaryOperator &I) { | ||||||||
| 1455 | assert(I.getOpcode() == BinaryOperator::Xor && "Only for xor!")(static_cast <bool> (I.getOpcode() == BinaryOperator::Xor && "Only for xor!") ? void (0) : __assert_fail ("I.getOpcode() == BinaryOperator::Xor && \"Only for xor!\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1455 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1456 | Value *Cond, *X; | ||||||||
| 1457 | // As per complexity ordering, `xor` is not commutative here. | ||||||||
| 1458 | if (!match(&I, m_c_BinOp(m_OneUse(m_Value()), m_Value())) || | ||||||||
| 1459 | !match(I.getOperand(1), m_SExt(m_Value(Cond))) || | ||||||||
| 1460 | !Cond->getType()->isIntOrIntVectorTy(1) || | ||||||||
| 1461 | !match(I.getOperand(0), m_c_Add(m_SExt(m_Deferred(Cond)), m_Value(X)))) | ||||||||
| 1462 | return nullptr; | ||||||||
| 1463 | return SelectInst::Create(Cond, Builder.CreateNeg(X, X->getName() + ".neg"), | ||||||||
| 1464 | X); | ||||||||
| 1465 | } | ||||||||
| 1466 | |||||||||
| 1467 | /// This a limited reassociation for a special case (see above) where we are | ||||||||
| 1468 | /// checking if two values are either both NAN (unordered) or not-NAN (ordered). | ||||||||
| 1469 | /// This could be handled more generally in '-reassociation', but it seems like | ||||||||
| 1470 | /// an unlikely pattern for a large number of logic ops and fcmps. | ||||||||
| 1471 | static Instruction *reassociateFCmps(BinaryOperator &BO, | ||||||||
| 1472 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1473 | Instruction::BinaryOps Opcode = BO.getOpcode(); | ||||||||
| 1474 | assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&(static_cast <bool> ((Opcode == Instruction::And || Opcode == Instruction::Or) && "Expecting and/or op for fcmp transform" ) ? void (0) : __assert_fail ("(Opcode == Instruction::And || Opcode == Instruction::Or) && \"Expecting and/or op for fcmp transform\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1475 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1475 | "Expecting and/or op for fcmp transform")(static_cast <bool> ((Opcode == Instruction::And || Opcode == Instruction::Or) && "Expecting and/or op for fcmp transform" ) ? void (0) : __assert_fail ("(Opcode == Instruction::And || Opcode == Instruction::Or) && \"Expecting and/or op for fcmp transform\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1475 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1476 | |||||||||
| 1477 | // There are 4 commuted variants of the pattern. Canonicalize operands of this | ||||||||
| 1478 | // logic op so an fcmp is operand 0 and a matching logic op is operand 1. | ||||||||
| 1479 | Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1), *X; | ||||||||
| 1480 | FCmpInst::Predicate Pred; | ||||||||
| 1481 | if (match(Op1, m_FCmp(Pred, m_Value(), m_AnyZeroFP()))) | ||||||||
| 1482 | std::swap(Op0, Op1); | ||||||||
| 1483 | |||||||||
| 1484 | // Match inner binop and the predicate for combining 2 NAN checks into 1. | ||||||||
| 1485 | Value *BO10, *BO11; | ||||||||
| 1486 | FCmpInst::Predicate NanPred = Opcode == Instruction::And ? FCmpInst::FCMP_ORD | ||||||||
| 1487 | : FCmpInst::FCMP_UNO; | ||||||||
| 1488 | if (!match(Op0, m_FCmp(Pred, m_Value(X), m_AnyZeroFP())) || Pred != NanPred || | ||||||||
| 1489 | !match(Op1, m_BinOp(Opcode, m_Value(BO10), m_Value(BO11)))) | ||||||||
| 1490 | return nullptr; | ||||||||
| 1491 | |||||||||
| 1492 | // The inner logic op must have a matching fcmp operand. | ||||||||
| 1493 | Value *Y; | ||||||||
| 1494 | if (!match(BO10, m_FCmp(Pred, m_Value(Y), m_AnyZeroFP())) || | ||||||||
| 1495 | Pred != NanPred || X->getType() != Y->getType()) | ||||||||
| 1496 | std::swap(BO10, BO11); | ||||||||
| 1497 | |||||||||
| 1498 | if (!match(BO10, m_FCmp(Pred, m_Value(Y), m_AnyZeroFP())) || | ||||||||
| 1499 | Pred != NanPred || X->getType() != Y->getType()) | ||||||||
| 1500 | return nullptr; | ||||||||
| 1501 | |||||||||
| 1502 | // and (fcmp ord X, 0), (and (fcmp ord Y, 0), Z) --> and (fcmp ord X, Y), Z | ||||||||
| 1503 | // or (fcmp uno X, 0), (or (fcmp uno Y, 0), Z) --> or (fcmp uno X, Y), Z | ||||||||
| 1504 | Value *NewFCmp = Builder.CreateFCmp(Pred, X, Y); | ||||||||
| 1505 | if (auto *NewFCmpInst = dyn_cast<FCmpInst>(NewFCmp)) { | ||||||||
| 1506 | // Intersect FMF from the 2 source fcmps. | ||||||||
| 1507 | NewFCmpInst->copyIRFlags(Op0); | ||||||||
| 1508 | NewFCmpInst->andIRFlags(BO10); | ||||||||
| 1509 | } | ||||||||
| 1510 | return BinaryOperator::Create(Opcode, NewFCmp, BO11); | ||||||||
| 1511 | } | ||||||||
| 1512 | |||||||||
| 1513 | /// Match variations of De Morgan's Laws: | ||||||||
| 1514 | /// (~A & ~B) == (~(A | B)) | ||||||||
| 1515 | /// (~A | ~B) == (~(A & B)) | ||||||||
| 1516 | static Instruction *matchDeMorgansLaws(BinaryOperator &I, | ||||||||
| 1517 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1518 | const Instruction::BinaryOps Opcode = I.getOpcode(); | ||||||||
| 1519 | assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&(static_cast <bool> ((Opcode == Instruction::And || Opcode == Instruction::Or) && "Trying to match De Morgan's Laws with something other than and/or" ) ? void (0) : __assert_fail ("(Opcode == Instruction::And || Opcode == Instruction::Or) && \"Trying to match De Morgan's Laws with something other than and/or\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1520 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1520 | "Trying to match De Morgan's Laws with something other than and/or")(static_cast <bool> ((Opcode == Instruction::And || Opcode == Instruction::Or) && "Trying to match De Morgan's Laws with something other than and/or" ) ? void (0) : __assert_fail ("(Opcode == Instruction::And || Opcode == Instruction::Or) && \"Trying to match De Morgan's Laws with something other than and/or\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1520 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1521 | |||||||||
| 1522 | // Flip the logic operation. | ||||||||
| 1523 | const Instruction::BinaryOps FlippedOpcode = | ||||||||
| 1524 | (Opcode == Instruction::And) ? Instruction::Or : Instruction::And; | ||||||||
| 1525 | |||||||||
| 1526 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1527 | Value *A, *B; | ||||||||
| 1528 | if (match(Op0, m_OneUse(m_Not(m_Value(A)))) && | ||||||||
| 1529 | match(Op1, m_OneUse(m_Not(m_Value(B)))) && | ||||||||
| 1530 | !InstCombiner::isFreeToInvert(A, A->hasOneUse()) && | ||||||||
| 1531 | !InstCombiner::isFreeToInvert(B, B->hasOneUse())) { | ||||||||
| 1532 | Value *AndOr = | ||||||||
| 1533 | Builder.CreateBinOp(FlippedOpcode, A, B, I.getName() + ".demorgan"); | ||||||||
| 1534 | return BinaryOperator::CreateNot(AndOr); | ||||||||
| 1535 | } | ||||||||
| 1536 | |||||||||
| 1537 | // The 'not' ops may require reassociation. | ||||||||
| 1538 | // (A & ~B) & ~C --> A & ~(B | C) | ||||||||
| 1539 | // (~B & A) & ~C --> A & ~(B | C) | ||||||||
| 1540 | // (A | ~B) | ~C --> A | ~(B & C) | ||||||||
| 1541 | // (~B | A) | ~C --> A | ~(B & C) | ||||||||
| 1542 | Value *C; | ||||||||
| 1543 | if (match(Op0, m_OneUse(m_c_BinOp(Opcode, m_Value(A), m_Not(m_Value(B))))) && | ||||||||
| 1544 | match(Op1, m_Not(m_Value(C)))) { | ||||||||
| 1545 | Value *FlippedBO = Builder.CreateBinOp(FlippedOpcode, B, C); | ||||||||
| 1546 | return BinaryOperator::Create(Opcode, A, Builder.CreateNot(FlippedBO)); | ||||||||
| 1547 | } | ||||||||
| 1548 | |||||||||
| 1549 | return nullptr; | ||||||||
| 1550 | } | ||||||||
| 1551 | |||||||||
| 1552 | bool InstCombinerImpl::shouldOptimizeCast(CastInst *CI) { | ||||||||
| 1553 | Value *CastSrc = CI->getOperand(0); | ||||||||
| 1554 | |||||||||
| 1555 | // Noop casts and casts of constants should be eliminated trivially. | ||||||||
| 1556 | if (CI->getSrcTy() == CI->getDestTy() || isa<Constant>(CastSrc)) | ||||||||
| 1557 | return false; | ||||||||
| 1558 | |||||||||
| 1559 | // If this cast is paired with another cast that can be eliminated, we prefer | ||||||||
| 1560 | // to have it eliminated. | ||||||||
| 1561 | if (const auto *PrecedingCI = dyn_cast<CastInst>(CastSrc)) | ||||||||
| 1562 | if (isEliminableCastPair(PrecedingCI, CI)) | ||||||||
| 1563 | return false; | ||||||||
| 1564 | |||||||||
| 1565 | return true; | ||||||||
| 1566 | } | ||||||||
| 1567 | |||||||||
| 1568 | /// Fold {and,or,xor} (cast X), C. | ||||||||
| 1569 | static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast, | ||||||||
| 1570 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1571 | Constant *C = dyn_cast<Constant>(Logic.getOperand(1)); | ||||||||
| 1572 | if (!C) | ||||||||
| 1573 | return nullptr; | ||||||||
| 1574 | |||||||||
| 1575 | auto LogicOpc = Logic.getOpcode(); | ||||||||
| 1576 | Type *DestTy = Logic.getType(); | ||||||||
| 1577 | Type *SrcTy = Cast->getSrcTy(); | ||||||||
| 1578 | |||||||||
| 1579 | // Move the logic operation ahead of a zext or sext if the constant is | ||||||||
| 1580 | // unchanged in the smaller source type. Performing the logic in a smaller | ||||||||
| 1581 | // type may provide more information to later folds, and the smaller logic | ||||||||
| 1582 | // instruction may be cheaper (particularly in the case of vectors). | ||||||||
| 1583 | Value *X; | ||||||||
| 1584 | if (match(Cast, m_OneUse(m_ZExt(m_Value(X))))) { | ||||||||
| 1585 | Constant *TruncC = ConstantExpr::getTrunc(C, SrcTy); | ||||||||
| 1586 | Constant *ZextTruncC = ConstantExpr::getZExt(TruncC, DestTy); | ||||||||
| 1587 | if (ZextTruncC == C) { | ||||||||
| 1588 | // LogicOpc (zext X), C --> zext (LogicOpc X, C) | ||||||||
| 1589 | Value *NewOp = Builder.CreateBinOp(LogicOpc, X, TruncC); | ||||||||
| 1590 | return new ZExtInst(NewOp, DestTy); | ||||||||
| 1591 | } | ||||||||
| 1592 | } | ||||||||
| 1593 | |||||||||
| 1594 | if (match(Cast, m_OneUse(m_SExt(m_Value(X))))) { | ||||||||
| 1595 | Constant *TruncC = ConstantExpr::getTrunc(C, SrcTy); | ||||||||
| 1596 | Constant *SextTruncC = ConstantExpr::getSExt(TruncC, DestTy); | ||||||||
| 1597 | if (SextTruncC == C) { | ||||||||
| 1598 | // LogicOpc (sext X), C --> sext (LogicOpc X, C) | ||||||||
| 1599 | Value *NewOp = Builder.CreateBinOp(LogicOpc, X, TruncC); | ||||||||
| 1600 | return new SExtInst(NewOp, DestTy); | ||||||||
| 1601 | } | ||||||||
| 1602 | } | ||||||||
| 1603 | |||||||||
| 1604 | return nullptr; | ||||||||
| 1605 | } | ||||||||
| 1606 | |||||||||
| 1607 | /// Fold {and,or,xor} (cast X), Y. | ||||||||
| 1608 | Instruction *InstCombinerImpl::foldCastedBitwiseLogic(BinaryOperator &I) { | ||||||||
| 1609 | auto LogicOpc = I.getOpcode(); | ||||||||
| 1610 | assert(I.isBitwiseLogicOp() && "Unexpected opcode for bitwise logic folding")(static_cast <bool> (I.isBitwiseLogicOp() && "Unexpected opcode for bitwise logic folding" ) ? void (0) : __assert_fail ("I.isBitwiseLogicOp() && \"Unexpected opcode for bitwise logic folding\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1610 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1611 | |||||||||
| 1612 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1613 | CastInst *Cast0 = dyn_cast<CastInst>(Op0); | ||||||||
| 1614 | if (!Cast0) | ||||||||
| 1615 | return nullptr; | ||||||||
| 1616 | |||||||||
| 1617 | // This must be a cast from an integer or integer vector source type to allow | ||||||||
| 1618 | // transformation of the logic operation to the source type. | ||||||||
| 1619 | Type *DestTy = I.getType(); | ||||||||
| 1620 | Type *SrcTy = Cast0->getSrcTy(); | ||||||||
| 1621 | if (!SrcTy->isIntOrIntVectorTy()) | ||||||||
| 1622 | return nullptr; | ||||||||
| 1623 | |||||||||
| 1624 | if (Instruction *Ret = foldLogicCastConstant(I, Cast0, Builder)) | ||||||||
| 1625 | return Ret; | ||||||||
| 1626 | |||||||||
| 1627 | CastInst *Cast1 = dyn_cast<CastInst>(Op1); | ||||||||
| 1628 | if (!Cast1) | ||||||||
| 1629 | return nullptr; | ||||||||
| 1630 | |||||||||
| 1631 | // Both operands of the logic operation are casts. The casts must be the | ||||||||
| 1632 | // same kind for reduction. | ||||||||
| 1633 | Instruction::CastOps CastOpcode = Cast0->getOpcode(); | ||||||||
| 1634 | if (CastOpcode != Cast1->getOpcode()) | ||||||||
| 1635 | return nullptr; | ||||||||
| 1636 | |||||||||
| 1637 | // If the source types do not match, but the casts are matching extends, we | ||||||||
| 1638 | // can still narrow the logic op. | ||||||||
| 1639 | if (SrcTy != Cast1->getSrcTy()) { | ||||||||
| 1640 | Value *X, *Y; | ||||||||
| 1641 | if (match(Cast0, m_OneUse(m_ZExtOrSExt(m_Value(X)))) && | ||||||||
| 1642 | match(Cast1, m_OneUse(m_ZExtOrSExt(m_Value(Y))))) { | ||||||||
| 1643 | // Cast the narrower source to the wider source type. | ||||||||
| 1644 | unsigned XNumBits = X->getType()->getScalarSizeInBits(); | ||||||||
| 1645 | unsigned YNumBits = Y->getType()->getScalarSizeInBits(); | ||||||||
| 1646 | if (XNumBits < YNumBits) | ||||||||
| 1647 | X = Builder.CreateCast(CastOpcode, X, Y->getType()); | ||||||||
| 1648 | else | ||||||||
| 1649 | Y = Builder.CreateCast(CastOpcode, Y, X->getType()); | ||||||||
| 1650 | // Do the logic op in the intermediate width, then widen more. | ||||||||
| 1651 | Value *NarrowLogic = Builder.CreateBinOp(LogicOpc, X, Y); | ||||||||
| 1652 | return CastInst::Create(CastOpcode, NarrowLogic, DestTy); | ||||||||
| 1653 | } | ||||||||
| 1654 | |||||||||
| 1655 | // Give up for other cast opcodes. | ||||||||
| 1656 | return nullptr; | ||||||||
| 1657 | } | ||||||||
| 1658 | |||||||||
| 1659 | Value *Cast0Src = Cast0->getOperand(0); | ||||||||
| 1660 | Value *Cast1Src = Cast1->getOperand(0); | ||||||||
| 1661 | |||||||||
| 1662 | // fold logic(cast(A), cast(B)) -> cast(logic(A, B)) | ||||||||
| 1663 | if ((Cast0->hasOneUse() || Cast1->hasOneUse()) && | ||||||||
| 1664 | shouldOptimizeCast(Cast0) && shouldOptimizeCast(Cast1)) { | ||||||||
| 1665 | Value *NewOp = Builder.CreateBinOp(LogicOpc, Cast0Src, Cast1Src, | ||||||||
| 1666 | I.getName()); | ||||||||
| 1667 | return CastInst::Create(CastOpcode, NewOp, DestTy); | ||||||||
| 1668 | } | ||||||||
| 1669 | |||||||||
| 1670 | // For now, only 'and'/'or' have optimizations after this. | ||||||||
| 1671 | if (LogicOpc == Instruction::Xor) | ||||||||
| 1672 | return nullptr; | ||||||||
| 1673 | |||||||||
| 1674 | // If this is logic(cast(icmp), cast(icmp)), try to fold this even if the | ||||||||
| 1675 | // cast is otherwise not optimizable. This happens for vector sexts. | ||||||||
| 1676 | ICmpInst *ICmp0 = dyn_cast<ICmpInst>(Cast0Src); | ||||||||
| 1677 | ICmpInst *ICmp1 = dyn_cast<ICmpInst>(Cast1Src); | ||||||||
| 1678 | if (ICmp0 && ICmp1) { | ||||||||
| 1679 | if (Value *Res = | ||||||||
| 1680 | foldAndOrOfICmps(ICmp0, ICmp1, I, LogicOpc == Instruction::And)) | ||||||||
| 1681 | return CastInst::Create(CastOpcode, Res, DestTy); | ||||||||
| 1682 | return nullptr; | ||||||||
| 1683 | } | ||||||||
| 1684 | |||||||||
| 1685 | // If this is logic(cast(fcmp), cast(fcmp)), try to fold this even if the | ||||||||
| 1686 | // cast is otherwise not optimizable. This happens for vector sexts. | ||||||||
| 1687 | FCmpInst *FCmp0 = dyn_cast<FCmpInst>(Cast0Src); | ||||||||
| 1688 | FCmpInst *FCmp1 = dyn_cast<FCmpInst>(Cast1Src); | ||||||||
| 1689 | if (FCmp0 && FCmp1) | ||||||||
| 1690 | if (Value *R = foldLogicOfFCmps(FCmp0, FCmp1, LogicOpc == Instruction::And)) | ||||||||
| 1691 | return CastInst::Create(CastOpcode, R, DestTy); | ||||||||
| 1692 | |||||||||
| 1693 | return nullptr; | ||||||||
| 1694 | } | ||||||||
| 1695 | |||||||||
| 1696 | static Instruction *foldAndToXor(BinaryOperator &I, | ||||||||
| 1697 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1698 | assert(I.getOpcode() == Instruction::And)(static_cast <bool> (I.getOpcode() == Instruction::And) ? void (0) : __assert_fail ("I.getOpcode() == Instruction::And" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1698 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1699 | Value *Op0 = I.getOperand(0); | ||||||||
| 1700 | Value *Op1 = I.getOperand(1); | ||||||||
| 1701 | Value *A, *B; | ||||||||
| 1702 | |||||||||
| 1703 | // Operand complexity canonicalization guarantees that the 'or' is Op0. | ||||||||
| 1704 | // (A | B) & ~(A & B) --> A ^ B | ||||||||
| 1705 | // (A | B) & ~(B & A) --> A ^ B | ||||||||
| 1706 | if (match(&I, m_BinOp(m_Or(m_Value(A), m_Value(B)), | ||||||||
| 1707 | m_Not(m_c_And(m_Deferred(A), m_Deferred(B)))))) | ||||||||
| 1708 | return BinaryOperator::CreateXor(A, B); | ||||||||
| 1709 | |||||||||
| 1710 | // (A | ~B) & (~A | B) --> ~(A ^ B) | ||||||||
| 1711 | // (A | ~B) & (B | ~A) --> ~(A ^ B) | ||||||||
| 1712 | // (~B | A) & (~A | B) --> ~(A ^ B) | ||||||||
| 1713 | // (~B | A) & (B | ~A) --> ~(A ^ B) | ||||||||
| 1714 | if (Op0->hasOneUse() || Op1->hasOneUse()) | ||||||||
| 1715 | if (match(&I, m_BinOp(m_c_Or(m_Value(A), m_Not(m_Value(B))), | ||||||||
| 1716 | m_c_Or(m_Not(m_Deferred(A)), m_Deferred(B))))) | ||||||||
| 1717 | return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); | ||||||||
| 1718 | |||||||||
| 1719 | return nullptr; | ||||||||
| 1720 | } | ||||||||
| 1721 | |||||||||
| 1722 | static Instruction *foldOrToXor(BinaryOperator &I, | ||||||||
| 1723 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1724 | assert(I.getOpcode() == Instruction::Or)(static_cast <bool> (I.getOpcode() == Instruction::Or) ? void (0) : __assert_fail ("I.getOpcode() == Instruction::Or" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1724 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1725 | Value *Op0 = I.getOperand(0); | ||||||||
| 1726 | Value *Op1 = I.getOperand(1); | ||||||||
| 1727 | Value *A, *B; | ||||||||
| 1728 | |||||||||
| 1729 | // Operand complexity canonicalization guarantees that the 'and' is Op0. | ||||||||
| 1730 | // (A & B) | ~(A | B) --> ~(A ^ B) | ||||||||
| 1731 | // (A & B) | ~(B | A) --> ~(A ^ B) | ||||||||
| 1732 | if (Op0->hasOneUse() || Op1->hasOneUse()) | ||||||||
| 1733 | if (match(Op0, m_And(m_Value(A), m_Value(B))) && | ||||||||
| 1734 | match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) | ||||||||
| 1735 | return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); | ||||||||
| 1736 | |||||||||
| 1737 | // Operand complexity canonicalization guarantees that the 'xor' is Op0. | ||||||||
| 1738 | // (A ^ B) | ~(A | B) --> ~(A & B) | ||||||||
| 1739 | // (A ^ B) | ~(B | A) --> ~(A & B) | ||||||||
| 1740 | if (Op0->hasOneUse() || Op1->hasOneUse()) | ||||||||
| 1741 | if (match(Op0, m_Xor(m_Value(A), m_Value(B))) && | ||||||||
| 1742 | match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) | ||||||||
| 1743 | return BinaryOperator::CreateNot(Builder.CreateAnd(A, B)); | ||||||||
| 1744 | |||||||||
| 1745 | // (A & ~B) | (~A & B) --> A ^ B | ||||||||
| 1746 | // (A & ~B) | (B & ~A) --> A ^ B | ||||||||
| 1747 | // (~B & A) | (~A & B) --> A ^ B | ||||||||
| 1748 | // (~B & A) | (B & ~A) --> A ^ B | ||||||||
| 1749 | if (match(Op0, m_c_And(m_Value(A), m_Not(m_Value(B)))) && | ||||||||
| 1750 | match(Op1, m_c_And(m_Not(m_Specific(A)), m_Specific(B)))) | ||||||||
| 1751 | return BinaryOperator::CreateXor(A, B); | ||||||||
| 1752 | |||||||||
| 1753 | return nullptr; | ||||||||
| 1754 | } | ||||||||
| 1755 | |||||||||
| 1756 | /// Return true if a constant shift amount is always less than the specified | ||||||||
| 1757 | /// bit-width. If not, the shift could create poison in the narrower type. | ||||||||
| 1758 | static bool canNarrowShiftAmt(Constant *C, unsigned BitWidth) { | ||||||||
| 1759 | APInt Threshold(C->getType()->getScalarSizeInBits(), BitWidth); | ||||||||
| 1760 | return match(C, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, Threshold)); | ||||||||
| 1761 | } | ||||||||
| 1762 | |||||||||
| 1763 | /// Try to use narrower ops (sink zext ops) for an 'and' with binop operand and | ||||||||
| 1764 | /// a common zext operand: and (binop (zext X), C), (zext X). | ||||||||
| 1765 | Instruction *InstCombinerImpl::narrowMaskedBinOp(BinaryOperator &And) { | ||||||||
| 1766 | // This transform could also apply to {or, and, xor}, but there are better | ||||||||
| 1767 | // folds for those cases, so we don't expect those patterns here. AShr is not | ||||||||
| 1768 | // handled because it should always be transformed to LShr in this sequence. | ||||||||
| 1769 | // The subtract transform is different because it has a constant on the left. | ||||||||
| 1770 | // Add/mul commute the constant to RHS; sub with constant RHS becomes add. | ||||||||
| 1771 | Value *Op0 = And.getOperand(0), *Op1 = And.getOperand(1); | ||||||||
| 1772 | Constant *C; | ||||||||
| 1773 | if (!match(Op0, m_OneUse(m_Add(m_Specific(Op1), m_Constant(C)))) && | ||||||||
| 1774 | !match(Op0, m_OneUse(m_Mul(m_Specific(Op1), m_Constant(C)))) && | ||||||||
| 1775 | !match(Op0, m_OneUse(m_LShr(m_Specific(Op1), m_Constant(C)))) && | ||||||||
| 1776 | !match(Op0, m_OneUse(m_Shl(m_Specific(Op1), m_Constant(C)))) && | ||||||||
| 1777 | !match(Op0, m_OneUse(m_Sub(m_Constant(C), m_Specific(Op1))))) | ||||||||
| 1778 | return nullptr; | ||||||||
| 1779 | |||||||||
| 1780 | Value *X; | ||||||||
| 1781 | if (!match(Op1, m_ZExt(m_Value(X))) || Op1->hasNUsesOrMore(3)) | ||||||||
| 1782 | return nullptr; | ||||||||
| 1783 | |||||||||
| 1784 | Type *Ty = And.getType(); | ||||||||
| 1785 | if (!isa<VectorType>(Ty) && !shouldChangeType(Ty, X->getType())) | ||||||||
| 1786 | return nullptr; | ||||||||
| 1787 | |||||||||
| 1788 | // If we're narrowing a shift, the shift amount must be safe (less than the | ||||||||
| 1789 | // width) in the narrower type. If the shift amount is greater, instsimplify | ||||||||
| 1790 | // usually handles that case, but we can't guarantee/assert it. | ||||||||
| 1791 | Instruction::BinaryOps Opc = cast<BinaryOperator>(Op0)->getOpcode(); | ||||||||
| 1792 | if (Opc == Instruction::LShr || Opc == Instruction::Shl) | ||||||||
| 1793 | if (!canNarrowShiftAmt(C, X->getType()->getScalarSizeInBits())) | ||||||||
| 1794 | return nullptr; | ||||||||
| 1795 | |||||||||
| 1796 | // and (sub C, (zext X)), (zext X) --> zext (and (sub C', X), X) | ||||||||
| 1797 | // and (binop (zext X), C), (zext X) --> zext (and (binop X, C'), X) | ||||||||
| 1798 | Value *NewC = ConstantExpr::getTrunc(C, X->getType()); | ||||||||
| 1799 | Value *NewBO = Opc == Instruction::Sub ? Builder.CreateBinOp(Opc, NewC, X) | ||||||||
| 1800 | : Builder.CreateBinOp(Opc, X, NewC); | ||||||||
| 1801 | return new ZExtInst(Builder.CreateAnd(NewBO, X), Ty); | ||||||||
| 1802 | } | ||||||||
| 1803 | |||||||||
| 1804 | /// Try folding relatively complex patterns for both And and Or operations | ||||||||
| 1805 | /// with all And and Or swapped. | ||||||||
| 1806 | static Instruction *foldComplexAndOrPatterns(BinaryOperator &I, | ||||||||
| 1807 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1808 | const Instruction::BinaryOps Opcode = I.getOpcode(); | ||||||||
| 1809 | assert(Opcode == Instruction::And || Opcode == Instruction::Or)(static_cast <bool> (Opcode == Instruction::And || Opcode == Instruction::Or) ? void (0) : __assert_fail ("Opcode == Instruction::And || Opcode == Instruction::Or" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 1809 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1810 | |||||||||
| 1811 | // Flip the logic operation. | ||||||||
| 1812 | const Instruction::BinaryOps FlippedOpcode = | ||||||||
| 1813 | (Opcode == Instruction::And) ? Instruction::Or : Instruction::And; | ||||||||
| 1814 | |||||||||
| 1815 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1816 | Value *A, *B, *C, *X, *Y, *Dummy; | ||||||||
| 1817 | |||||||||
| 1818 | // Match following expressions: | ||||||||
| 1819 | // (~(A | B) & C) | ||||||||
| 1820 | // (~(A & B) | C) | ||||||||
| 1821 | // Captures X = ~(A | B) or ~(A & B) | ||||||||
| 1822 | const auto matchNotOrAnd = | ||||||||
| 1823 | [Opcode, FlippedOpcode](Value *Op, auto m_A, auto m_B, auto m_C, | ||||||||
| 1824 | Value *&X, bool CountUses = false) -> bool { | ||||||||
| 1825 | if (CountUses && !Op->hasOneUse()) | ||||||||
| 1826 | return false; | ||||||||
| 1827 | |||||||||
| 1828 | if (match(Op, m_c_BinOp(FlippedOpcode, | ||||||||
| 1829 | m_CombineAnd(m_Value(X), | ||||||||
| 1830 | m_Not(m_c_BinOp(Opcode, m_A, m_B))), | ||||||||
| 1831 | m_C))) | ||||||||
| 1832 | return !CountUses || X->hasOneUse(); | ||||||||
| 1833 | |||||||||
| 1834 | return false; | ||||||||
| 1835 | }; | ||||||||
| 1836 | |||||||||
| 1837 | // (~(A | B) & C) | ... --> ... | ||||||||
| 1838 | // (~(A & B) | C) & ... --> ... | ||||||||
| 1839 | // TODO: One use checks are conservative. We just need to check that a total | ||||||||
| 1840 | // number of multiple used values does not exceed reduction | ||||||||
| 1841 | // in operations. | ||||||||
| 1842 | if (matchNotOrAnd(Op0, m_Value(A), m_Value(B), m_Value(C), X)) { | ||||||||
| 1843 | // (~(A | B) & C) | (~(A | C) & B) --> (B ^ C) & ~A | ||||||||
| 1844 | // (~(A & B) | C) & (~(A & C) | B) --> ~((B ^ C) & A) | ||||||||
| 1845 | if (matchNotOrAnd(Op1, m_Specific(A), m_Specific(C), m_Specific(B), Dummy, | ||||||||
| 1846 | true)) { | ||||||||
| 1847 | Value *Xor = Builder.CreateXor(B, C); | ||||||||
| 1848 | return (Opcode == Instruction::Or) | ||||||||
| 1849 | ? BinaryOperator::CreateAnd(Xor, Builder.CreateNot(A)) | ||||||||
| 1850 | : BinaryOperator::CreateNot(Builder.CreateAnd(Xor, A)); | ||||||||
| 1851 | } | ||||||||
| 1852 | |||||||||
| 1853 | // (~(A | B) & C) | (~(B | C) & A) --> (A ^ C) & ~B | ||||||||
| 1854 | // (~(A & B) | C) & (~(B & C) | A) --> ~((A ^ C) & B) | ||||||||
| 1855 | if (matchNotOrAnd(Op1, m_Specific(B), m_Specific(C), m_Specific(A), Dummy, | ||||||||
| 1856 | true)) { | ||||||||
| 1857 | Value *Xor = Builder.CreateXor(A, C); | ||||||||
| 1858 | return (Opcode == Instruction::Or) | ||||||||
| 1859 | ? BinaryOperator::CreateAnd(Xor, Builder.CreateNot(B)) | ||||||||
| 1860 | : BinaryOperator::CreateNot(Builder.CreateAnd(Xor, B)); | ||||||||
| 1861 | } | ||||||||
| 1862 | |||||||||
| 1863 | // (~(A | B) & C) | ~(A | C) --> ~((B & C) | A) | ||||||||
| 1864 | // (~(A & B) | C) & ~(A & C) --> ~((B | C) & A) | ||||||||
| 1865 | if (match(Op1, m_OneUse(m_Not(m_OneUse( | ||||||||
| 1866 | m_c_BinOp(Opcode, m_Specific(A), m_Specific(C))))))) | ||||||||
| 1867 | return BinaryOperator::CreateNot(Builder.CreateBinOp( | ||||||||
| 1868 | Opcode, Builder.CreateBinOp(FlippedOpcode, B, C), A)); | ||||||||
| 1869 | |||||||||
| 1870 | // (~(A | B) & C) | ~(B | C) --> ~((A & C) | B) | ||||||||
| 1871 | // (~(A & B) | C) & ~(B & C) --> ~((A | C) & B) | ||||||||
| 1872 | if (match(Op1, m_OneUse(m_Not(m_OneUse( | ||||||||
| 1873 | m_c_BinOp(Opcode, m_Specific(B), m_Specific(C))))))) | ||||||||
| 1874 | return BinaryOperator::CreateNot(Builder.CreateBinOp( | ||||||||
| 1875 | Opcode, Builder.CreateBinOp(FlippedOpcode, A, C), B)); | ||||||||
| 1876 | |||||||||
| 1877 | // (~(A | B) & C) | ~(C | (A ^ B)) --> ~((A | B) & (C | (A ^ B))) | ||||||||
| 1878 | // Note, the pattern with swapped and/or is not handled because the | ||||||||
| 1879 | // result is more undefined than a source: | ||||||||
| 1880 | // (~(A & B) | C) & ~(C & (A ^ B)) --> (A ^ B ^ C) | ~(A | C) is invalid. | ||||||||
| 1881 | if (Opcode == Instruction::Or && Op0->hasOneUse() && | ||||||||
| 1882 | match(Op1, m_OneUse(m_Not(m_CombineAnd( | ||||||||
| 1883 | m_Value(Y), | ||||||||
| 1884 | m_c_BinOp(Opcode, m_Specific(C), | ||||||||
| 1885 | m_c_Xor(m_Specific(A), m_Specific(B)))))))) { | ||||||||
| 1886 | // X = ~(A | B) | ||||||||
| 1887 | // Y = (C | (A ^ B) | ||||||||
| 1888 | Value *Or = cast<BinaryOperator>(X)->getOperand(0); | ||||||||
| 1889 | return BinaryOperator::CreateNot(Builder.CreateAnd(Or, Y)); | ||||||||
| 1890 | } | ||||||||
| 1891 | } | ||||||||
| 1892 | |||||||||
| 1893 | // (~A & B & C) | ... --> ... | ||||||||
| 1894 | // (~A | B | C) | ... --> ... | ||||||||
| 1895 | // TODO: One use checks are conservative. We just need to check that a total | ||||||||
| 1896 | // number of multiple used values does not exceed reduction | ||||||||
| 1897 | // in operations. | ||||||||
| 1898 | if (match(Op0, | ||||||||
| 1899 | m_OneUse(m_c_BinOp(FlippedOpcode, | ||||||||
| 1900 | m_BinOp(FlippedOpcode, m_Value(B), m_Value(C)), | ||||||||
| 1901 | m_CombineAnd(m_Value(X), m_Not(m_Value(A)))))) || | ||||||||
| 1902 | match(Op0, m_OneUse(m_c_BinOp( | ||||||||
| 1903 | FlippedOpcode, | ||||||||
| 1904 | m_c_BinOp(FlippedOpcode, m_Value(C), | ||||||||
| 1905 | m_CombineAnd(m_Value(X), m_Not(m_Value(A)))), | ||||||||
| 1906 | m_Value(B))))) { | ||||||||
| 1907 | // X = ~A | ||||||||
| 1908 | // (~A & B & C) | ~(A | B | C) --> ~(A | (B ^ C)) | ||||||||
| 1909 | // (~A | B | C) & ~(A & B & C) --> (~A | (B ^ C)) | ||||||||
| 1910 | if (match(Op1, m_OneUse(m_Not(m_c_BinOp( | ||||||||
| 1911 | Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(B)), | ||||||||
| 1912 | m_Specific(C))))) || | ||||||||
| 1913 | match(Op1, m_OneUse(m_Not(m_c_BinOp( | ||||||||
| 1914 | Opcode, m_c_BinOp(Opcode, m_Specific(B), m_Specific(C)), | ||||||||
| 1915 | m_Specific(A))))) || | ||||||||
| 1916 | match(Op1, m_OneUse(m_Not(m_c_BinOp( | ||||||||
| 1917 | Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)), | ||||||||
| 1918 | m_Specific(B)))))) { | ||||||||
| 1919 | Value *Xor = Builder.CreateXor(B, C); | ||||||||
| 1920 | return (Opcode == Instruction::Or) | ||||||||
| 1921 | ? BinaryOperator::CreateNot(Builder.CreateOr(Xor, A)) | ||||||||
| 1922 | : BinaryOperator::CreateOr(Xor, X); | ||||||||
| 1923 | } | ||||||||
| 1924 | |||||||||
| 1925 | // (~A & B & C) | ~(A | B) --> (C | ~B) & ~A | ||||||||
| 1926 | // (~A | B | C) & ~(A & B) --> (C & ~B) | ~A | ||||||||
| 1927 | if (match(Op1, m_OneUse(m_Not(m_OneUse( | ||||||||
| 1928 | m_c_BinOp(Opcode, m_Specific(A), m_Specific(B))))))) | ||||||||
| 1929 | return BinaryOperator::Create( | ||||||||
| 1930 | FlippedOpcode, Builder.CreateBinOp(Opcode, C, Builder.CreateNot(B)), | ||||||||
| 1931 | X); | ||||||||
| 1932 | |||||||||
| 1933 | // (~A & B & C) | ~(A | C) --> (B | ~C) & ~A | ||||||||
| 1934 | // (~A | B | C) & ~(A & C) --> (B & ~C) | ~A | ||||||||
| 1935 | if (match(Op1, m_OneUse(m_Not(m_OneUse( | ||||||||
| 1936 | m_c_BinOp(Opcode, m_Specific(A), m_Specific(C))))))) | ||||||||
| 1937 | return BinaryOperator::Create( | ||||||||
| 1938 | FlippedOpcode, Builder.CreateBinOp(Opcode, B, Builder.CreateNot(C)), | ||||||||
| 1939 | X); | ||||||||
| 1940 | } | ||||||||
| 1941 | |||||||||
| 1942 | return nullptr; | ||||||||
| 1943 | } | ||||||||
| 1944 | |||||||||
| 1945 | /// Try to reassociate a pair of binops so that values with one use only are | ||||||||
| 1946 | /// part of the same instruction. This may enable folds that are limited with | ||||||||
| 1947 | /// multi-use restrictions and makes it more likely to match other patterns that | ||||||||
| 1948 | /// are looking for a common operand. | ||||||||
| 1949 | static Instruction *reassociateForUses(BinaryOperator &BO, | ||||||||
| 1950 | InstCombinerImpl::BuilderTy &Builder) { | ||||||||
| 1951 | Instruction::BinaryOps Opcode = BO.getOpcode(); | ||||||||
| 1952 | Value *X, *Y, *Z; | ||||||||
| 1953 | if (match(&BO, | ||||||||
| 1954 | m_c_BinOp(Opcode, m_OneUse(m_BinOp(Opcode, m_Value(X), m_Value(Y))), | ||||||||
| 1955 | m_OneUse(m_Value(Z))))) { | ||||||||
| 1956 | if (!isa<Constant>(X) && !isa<Constant>(Y) && !isa<Constant>(Z)) { | ||||||||
| 1957 | // (X op Y) op Z --> (Y op Z) op X | ||||||||
| 1958 | if (!X->hasOneUse()) { | ||||||||
| 1959 | Value *YZ = Builder.CreateBinOp(Opcode, Y, Z); | ||||||||
| 1960 | return BinaryOperator::Create(Opcode, YZ, X); | ||||||||
| 1961 | } | ||||||||
| 1962 | // (X op Y) op Z --> (X op Z) op Y | ||||||||
| 1963 | if (!Y->hasOneUse()) { | ||||||||
| 1964 | Value *XZ = Builder.CreateBinOp(Opcode, X, Z); | ||||||||
| 1965 | return BinaryOperator::Create(Opcode, XZ, Y); | ||||||||
| 1966 | } | ||||||||
| 1967 | } | ||||||||
| 1968 | } | ||||||||
| 1969 | |||||||||
| 1970 | return nullptr; | ||||||||
| 1971 | } | ||||||||
| 1972 | |||||||||
| 1973 | // Match | ||||||||
| 1974 | // (X + C2) | C | ||||||||
| 1975 | // (X + C2) ^ C | ||||||||
| 1976 | // (X + C2) & C | ||||||||
| 1977 | // and convert to do the bitwise logic first: | ||||||||
| 1978 | // (X | C) + C2 | ||||||||
| 1979 | // (X ^ C) + C2 | ||||||||
| 1980 | // (X & C) + C2 | ||||||||
| 1981 | // iff bits affected by logic op are lower than last bit affected by math op | ||||||||
| 1982 | static Instruction *canonicalizeLogicFirst(BinaryOperator &I, | ||||||||
| 1983 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1984 | Type *Ty = I.getType(); | ||||||||
| 1985 | Instruction::BinaryOps OpC = I.getOpcode(); | ||||||||
| 1986 | Value *Op0 = I.getOperand(0); | ||||||||
| 1987 | Value *Op1 = I.getOperand(1); | ||||||||
| 1988 | Value *X; | ||||||||
| 1989 | const APInt *C, *C2; | ||||||||
| 1990 | |||||||||
| 1991 | if (!(match(Op0, m_OneUse(m_Add(m_Value(X), m_APInt(C2)))) && | ||||||||
| 1992 | match(Op1, m_APInt(C)))) | ||||||||
| 1993 | return nullptr; | ||||||||
| 1994 | |||||||||
| 1995 | unsigned Width = Ty->getScalarSizeInBits(); | ||||||||
| 1996 | unsigned LastOneMath = Width - C2->countr_zero(); | ||||||||
| 1997 | |||||||||
| 1998 | switch (OpC) { | ||||||||
| 1999 | case Instruction::And: | ||||||||
| 2000 | if (C->countl_one() < LastOneMath) | ||||||||
| 2001 | return nullptr; | ||||||||
| 2002 | break; | ||||||||
| 2003 | case Instruction::Xor: | ||||||||
| 2004 | case Instruction::Or: | ||||||||
| 2005 | if (C->countl_zero() < LastOneMath) | ||||||||
| 2006 | return nullptr; | ||||||||
| 2007 | break; | ||||||||
| 2008 | default: | ||||||||
| 2009 | llvm_unreachable("Unexpected BinaryOp!")::llvm::llvm_unreachable_internal("Unexpected BinaryOp!", "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp" , 2009); | ||||||||
| 2010 | } | ||||||||
| 2011 | |||||||||
| 2012 | auto *Add = cast<BinaryOperator>(Op0); | ||||||||
| 2013 | Value *NewBinOp = Builder.CreateBinOp(OpC, X, ConstantInt::get(Ty, *C)); | ||||||||
| 2014 | return BinaryOperator::CreateWithCopiedFlags(Instruction::Add, NewBinOp, | ||||||||
| 2015 | ConstantInt::get(Ty, *C2), Add); | ||||||||
| 2016 | } | ||||||||
| 2017 | |||||||||
| 2018 | // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches | ||||||||
| 2019 | // here. We should standardize that construct where it is needed or choose some | ||||||||
| 2020 | // other way to ensure that commutated variants of patterns are not missed. | ||||||||
| 2021 | Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) { | ||||||||
| 2022 | Type *Ty = I.getType(); | ||||||||
| 2023 | |||||||||
| 2024 | if (Value *V = simplifyAndInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 2025 | SQ.getWithInstruction(&I))) | ||||||||
| 2026 | return replaceInstUsesWith(I, V); | ||||||||
| 2027 | |||||||||
| 2028 | if (SimplifyAssociativeOrCommutative(I)) | ||||||||
| 2029 | return &I; | ||||||||
| 2030 | |||||||||
| 2031 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 2032 | return X; | ||||||||
| 2033 | |||||||||
| 2034 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | ||||||||
| 2035 | return Phi; | ||||||||
| 2036 | |||||||||
| 2037 | // See if we can simplify any instructions used by the instruction whose sole | ||||||||
| 2038 | // purpose is to compute bits we don't care about. | ||||||||
| 2039 | if (SimplifyDemandedInstructionBits(I)) | ||||||||
| 2040 | return &I; | ||||||||
| 2041 | |||||||||
| 2042 | // Do this before using distributive laws to catch simple and/or/not patterns. | ||||||||
| 2043 | if (Instruction *Xor = foldAndToXor(I, Builder)) | ||||||||
| 2044 | return Xor; | ||||||||
| 2045 | |||||||||
| 2046 | if (Instruction *X = foldComplexAndOrPatterns(I, Builder)) | ||||||||
| 2047 | return X; | ||||||||
| 2048 | |||||||||
| 2049 | // (A|B)&(A|C) -> A|(B&C) etc | ||||||||
| 2050 | if (Value *V = foldUsingDistributiveLaws(I)) | ||||||||
| 2051 | return replaceInstUsesWith(I, V); | ||||||||
| 2052 | |||||||||
| 2053 | if (Value *V = SimplifyBSwap(I, Builder)) | ||||||||
| 2054 | return replaceInstUsesWith(I, V); | ||||||||
| 2055 | |||||||||
| 2056 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 2057 | |||||||||
| 2058 | Value *X, *Y; | ||||||||
| 2059 | if (match(Op0, m_OneUse(m_LogicalShift(m_One(), m_Value(X)))) && | ||||||||
| 2060 | match(Op1, m_One())) { | ||||||||
| 2061 | // (1 << X) & 1 --> zext(X == 0) | ||||||||
| 2062 | // (1 >> X) & 1 --> zext(X == 0) | ||||||||
| 2063 | Value *IsZero = Builder.CreateICmpEQ(X, ConstantInt::get(Ty, 0)); | ||||||||
| 2064 | return new ZExtInst(IsZero, Ty); | ||||||||
| 2065 | } | ||||||||
| 2066 | |||||||||
| 2067 | // (-(X & 1)) & Y --> (X & 1) == 0 ? 0 : Y | ||||||||
| 2068 | Value *Neg; | ||||||||
| 2069 | if (match(&I, | ||||||||
| 2070 | m_c_And(m_CombineAnd(m_Value(Neg), | ||||||||
| 2071 | m_OneUse(m_Neg(m_And(m_Value(), m_One())))), | ||||||||
| 2072 | m_Value(Y)))) { | ||||||||
| 2073 | Value *Cmp = Builder.CreateIsNull(Neg); | ||||||||
| 2074 | return SelectInst::Create(Cmp, ConstantInt::getNullValue(Ty), Y); | ||||||||
| 2075 | } | ||||||||
| 2076 | |||||||||
| 2077 | const APInt *C; | ||||||||
| 2078 | if (match(Op1, m_APInt(C))) { | ||||||||
| 2079 | const APInt *XorC; | ||||||||
| 2080 | if (match(Op0, m_OneUse(m_Xor(m_Value(X), m_APInt(XorC))))) { | ||||||||
| 2081 | // (X ^ C1) & C2 --> (X & C2) ^ (C1&C2) | ||||||||
| 2082 | Constant *NewC = ConstantInt::get(Ty, *C & *XorC); | ||||||||
| 2083 | Value *And = Builder.CreateAnd(X, Op1); | ||||||||
| 2084 | And->takeName(Op0); | ||||||||
| 2085 | return BinaryOperator::CreateXor(And, NewC); | ||||||||
| 2086 | } | ||||||||
| 2087 | |||||||||
| 2088 | const APInt *OrC; | ||||||||
| 2089 | if (match(Op0, m_OneUse(m_Or(m_Value(X), m_APInt(OrC))))) { | ||||||||
| 2090 | // (X | C1) & C2 --> (X & C2^(C1&C2)) | (C1&C2) | ||||||||
| 2091 | // NOTE: This reduces the number of bits set in the & mask, which | ||||||||
| 2092 | // can expose opportunities for store narrowing for scalars. | ||||||||
| 2093 | // NOTE: SimplifyDemandedBits should have already removed bits from C1 | ||||||||
| 2094 | // that aren't set in C2. Meaning we can replace (C1&C2) with C1 in | ||||||||
| 2095 | // above, but this feels safer. | ||||||||
| 2096 | APInt Together = *C & *OrC; | ||||||||
| 2097 | Value *And = Builder.CreateAnd(X, ConstantInt::get(Ty, Together ^ *C)); | ||||||||
| 2098 | And->takeName(Op0); | ||||||||
| 2099 | return BinaryOperator::CreateOr(And, ConstantInt::get(Ty, Together)); | ||||||||
| 2100 | } | ||||||||
| 2101 | |||||||||
| 2102 | unsigned Width = Ty->getScalarSizeInBits(); | ||||||||
| 2103 | const APInt *ShiftC; | ||||||||
| 2104 | if (match(Op0, m_OneUse(m_SExt(m_AShr(m_Value(X), m_APInt(ShiftC))))) && | ||||||||
| 2105 | ShiftC->ult(Width)) { | ||||||||
| 2106 | if (*C == APInt::getLowBitsSet(Width, Width - ShiftC->getZExtValue())) { | ||||||||
| 2107 | // We are clearing high bits that were potentially set by sext+ashr: | ||||||||
| 2108 | // and (sext (ashr X, ShiftC)), C --> lshr (sext X), ShiftC | ||||||||
| 2109 | Value *Sext = Builder.CreateSExt(X, Ty); | ||||||||
| 2110 | Constant *ShAmtC = ConstantInt::get(Ty, ShiftC->zext(Width)); | ||||||||
| 2111 | return BinaryOperator::CreateLShr(Sext, ShAmtC); | ||||||||
| 2112 | } | ||||||||
| 2113 | } | ||||||||
| 2114 | |||||||||
| 2115 | // If this 'and' clears the sign-bits added by ashr, replace with lshr: | ||||||||
| 2116 | // and (ashr X, ShiftC), C --> lshr X, ShiftC | ||||||||
| 2117 | if (match(Op0, m_AShr(m_Value(X), m_APInt(ShiftC))) && ShiftC->ult(Width) && | ||||||||
| 2118 | C->isMask(Width - ShiftC->getZExtValue())) | ||||||||
| 2119 | return BinaryOperator::CreateLShr(X, ConstantInt::get(Ty, *ShiftC)); | ||||||||
| 2120 | |||||||||
| 2121 | const APInt *AddC; | ||||||||
| 2122 | if (match(Op0, m_Add(m_Value(X), m_APInt(AddC)))) { | ||||||||
| 2123 | // If we add zeros to every bit below a mask, the add has no effect: | ||||||||
| 2124 | // (X + AddC) & LowMaskC --> X & LowMaskC | ||||||||
| 2125 | unsigned Ctlz = C->countl_zero(); | ||||||||
| 2126 | APInt LowMask(APInt::getLowBitsSet(Width, Width - Ctlz)); | ||||||||
| 2127 | if ((*AddC & LowMask).isZero()) | ||||||||
| 2128 | return BinaryOperator::CreateAnd(X, Op1); | ||||||||
| 2129 | |||||||||
| 2130 | // If we are masking the result of the add down to exactly one bit and | ||||||||
| 2131 | // the constant we are adding has no bits set below that bit, then the | ||||||||
| 2132 | // add is flipping a single bit. Example: | ||||||||
| 2133 | // (X + 4) & 4 --> (X & 4) ^ 4 | ||||||||
| 2134 | if (Op0->hasOneUse() && C->isPowerOf2() && (*AddC & (*C - 1)) == 0) { | ||||||||
| 2135 | assert((*C & *AddC) != 0 && "Expected common bit")(static_cast <bool> ((*C & *AddC) != 0 && "Expected common bit" ) ? void (0) : __assert_fail ("(*C & *AddC) != 0 && \"Expected common bit\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 2135 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2136 | Value *NewAnd = Builder.CreateAnd(X, Op1); | ||||||||
| 2137 | return BinaryOperator::CreateXor(NewAnd, Op1); | ||||||||
| 2138 | } | ||||||||
| 2139 | } | ||||||||
| 2140 | |||||||||
| 2141 | // ((C1 OP zext(X)) & C2) -> zext((C1 OP X) & C2) if C2 fits in the | ||||||||
| 2142 | // bitwidth of X and OP behaves well when given trunc(C1) and X. | ||||||||
| 2143 | auto isNarrowableBinOpcode = [](BinaryOperator *B) { | ||||||||
| 2144 | switch (B->getOpcode()) { | ||||||||
| 2145 | case Instruction::Xor: | ||||||||
| 2146 | case Instruction::Or: | ||||||||
| 2147 | case Instruction::Mul: | ||||||||
| 2148 | case Instruction::Add: | ||||||||
| 2149 | case Instruction::Sub: | ||||||||
| 2150 | return true; | ||||||||
| 2151 | default: | ||||||||
| 2152 | return false; | ||||||||
| 2153 | } | ||||||||
| 2154 | }; | ||||||||
| 2155 | BinaryOperator *BO; | ||||||||
| 2156 | if (match(Op0, m_OneUse(m_BinOp(BO))) && isNarrowableBinOpcode(BO)) { | ||||||||
| 2157 | Instruction::BinaryOps BOpcode = BO->getOpcode(); | ||||||||
| 2158 | Value *X; | ||||||||
| 2159 | const APInt *C1; | ||||||||
| 2160 | // TODO: The one-use restrictions could be relaxed a little if the AND | ||||||||
| 2161 | // is going to be removed. | ||||||||
| 2162 | // Try to narrow the 'and' and a binop with constant operand: | ||||||||
| 2163 | // and (bo (zext X), C1), C --> zext (and (bo X, TruncC1), TruncC) | ||||||||
| 2164 | if (match(BO, m_c_BinOp(m_OneUse(m_ZExt(m_Value(X))), m_APInt(C1))) && | ||||||||
| 2165 | C->isIntN(X->getType()->getScalarSizeInBits())) { | ||||||||
| 2166 | unsigned XWidth = X->getType()->getScalarSizeInBits(); | ||||||||
| 2167 | Constant *TruncC1 = ConstantInt::get(X->getType(), C1->trunc(XWidth)); | ||||||||
| 2168 | Value *BinOp = isa<ZExtInst>(BO->getOperand(0)) | ||||||||
| 2169 | ? Builder.CreateBinOp(BOpcode, X, TruncC1) | ||||||||
| 2170 | : Builder.CreateBinOp(BOpcode, TruncC1, X); | ||||||||
| 2171 | Constant *TruncC = ConstantInt::get(X->getType(), C->trunc(XWidth)); | ||||||||
| 2172 | Value *And = Builder.CreateAnd(BinOp, TruncC); | ||||||||
| 2173 | return new ZExtInst(And, Ty); | ||||||||
| 2174 | } | ||||||||
| 2175 | |||||||||
| 2176 | // Similar to above: if the mask matches the zext input width, then the | ||||||||
| 2177 | // 'and' can be eliminated, so we can truncate the other variable op: | ||||||||
| 2178 | // and (bo (zext X), Y), C --> zext (bo X, (trunc Y)) | ||||||||
| 2179 | if (isa<Instruction>(BO->getOperand(0)) && | ||||||||
| 2180 | match(BO->getOperand(0), m_OneUse(m_ZExt(m_Value(X)))) && | ||||||||
| 2181 | C->isMask(X->getType()->getScalarSizeInBits())) { | ||||||||
| 2182 | Y = BO->getOperand(1); | ||||||||
| 2183 | Value *TrY = Builder.CreateTrunc(Y, X->getType(), Y->getName() + ".tr"); | ||||||||
| 2184 | Value *NewBO = | ||||||||
| 2185 | Builder.CreateBinOp(BOpcode, X, TrY, BO->getName() + ".narrow"); | ||||||||
| 2186 | return new ZExtInst(NewBO, Ty); | ||||||||
| 2187 | } | ||||||||
| 2188 | // and (bo Y, (zext X)), C --> zext (bo (trunc Y), X) | ||||||||
| 2189 | if (isa<Instruction>(BO->getOperand(1)) && | ||||||||
| 2190 | match(BO->getOperand(1), m_OneUse(m_ZExt(m_Value(X)))) && | ||||||||
| 2191 | C->isMask(X->getType()->getScalarSizeInBits())) { | ||||||||
| 2192 | Y = BO->getOperand(0); | ||||||||
| 2193 | Value *TrY = Builder.CreateTrunc(Y, X->getType(), Y->getName() + ".tr"); | ||||||||
| 2194 | Value *NewBO = | ||||||||
| 2195 | Builder.CreateBinOp(BOpcode, TrY, X, BO->getName() + ".narrow"); | ||||||||
| 2196 | return new ZExtInst(NewBO, Ty); | ||||||||
| 2197 | } | ||||||||
| 2198 | } | ||||||||
| 2199 | |||||||||
| 2200 | // This is intentionally placed after the narrowing transforms for | ||||||||
| 2201 | // efficiency (transform directly to the narrow logic op if possible). | ||||||||
| 2202 | // If the mask is only needed on one incoming arm, push the 'and' op up. | ||||||||
| 2203 | if (match(Op0, m_OneUse(m_Xor(m_Value(X), m_Value(Y)))) || | ||||||||
| 2204 | match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) { | ||||||||
| 2205 | APInt NotAndMask(~(*C)); | ||||||||
| 2206 | BinaryOperator::BinaryOps BinOp = cast<BinaryOperator>(Op0)->getOpcode(); | ||||||||
| 2207 | if (MaskedValueIsZero(X, NotAndMask, 0, &I)) { | ||||||||
| 2208 | // Not masking anything out for the LHS, move mask to RHS. | ||||||||
| 2209 | // and ({x}or X, Y), C --> {x}or X, (and Y, C) | ||||||||
| 2210 | Value *NewRHS = Builder.CreateAnd(Y, Op1, Y->getName() + ".masked"); | ||||||||
| 2211 | return BinaryOperator::Create(BinOp, X, NewRHS); | ||||||||
| 2212 | } | ||||||||
| 2213 | if (!isa<Constant>(Y) && MaskedValueIsZero(Y, NotAndMask, 0, &I)) { | ||||||||
| 2214 | // Not masking anything out for the RHS, move mask to LHS. | ||||||||
| 2215 | // and ({x}or X, Y), C --> {x}or (and X, C), Y | ||||||||
| 2216 | Value *NewLHS = Builder.CreateAnd(X, Op1, X->getName() + ".masked"); | ||||||||
| 2217 | return BinaryOperator::Create(BinOp, NewLHS, Y); | ||||||||
| 2218 | } | ||||||||
| 2219 | } | ||||||||
| 2220 | |||||||||
| 2221 | // When the mask is a power-of-2 constant and op0 is a shifted-power-of-2 | ||||||||
| 2222 | // constant, test if the shift amount equals the offset bit index: | ||||||||
| 2223 | // (ShiftC << X) & C --> X == (log2(C) - log2(ShiftC)) ? C : 0 | ||||||||
| 2224 | // (ShiftC >> X) & C --> X == (log2(ShiftC) - log2(C)) ? C : 0 | ||||||||
| 2225 | if (C->isPowerOf2() && | ||||||||
| 2226 | match(Op0, m_OneUse(m_LogicalShift(m_Power2(ShiftC), m_Value(X))))) { | ||||||||
| 2227 | int Log2ShiftC = ShiftC->exactLogBase2(); | ||||||||
| 2228 | int Log2C = C->exactLogBase2(); | ||||||||
| 2229 | bool IsShiftLeft = | ||||||||
| 2230 | cast<BinaryOperator>(Op0)->getOpcode() == Instruction::Shl; | ||||||||
| 2231 | int BitNum = IsShiftLeft ? Log2C - Log2ShiftC : Log2ShiftC - Log2C; | ||||||||
| 2232 | assert(BitNum >= 0 && "Expected demanded bits to handle impossible mask")(static_cast <bool> (BitNum >= 0 && "Expected demanded bits to handle impossible mask" ) ? void (0) : __assert_fail ("BitNum >= 0 && \"Expected demanded bits to handle impossible mask\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 2232 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2233 | Value *Cmp = Builder.CreateICmpEQ(X, ConstantInt::get(Ty, BitNum)); | ||||||||
| 2234 | return SelectInst::Create(Cmp, ConstantInt::get(Ty, *C), | ||||||||
| 2235 | ConstantInt::getNullValue(Ty)); | ||||||||
| 2236 | } | ||||||||
| 2237 | |||||||||
| 2238 | Constant *C1, *C2; | ||||||||
| 2239 | const APInt *C3 = C; | ||||||||
| 2240 | Value *X; | ||||||||
| 2241 | if (C3->isPowerOf2()) { | ||||||||
| 2242 | Constant *Log2C3 = ConstantInt::get(Ty, C3->countr_zero()); | ||||||||
| 2243 | if (match(Op0, m_OneUse(m_LShr(m_Shl(m_ImmConstant(C1), m_Value(X)), | ||||||||
| 2244 | m_ImmConstant(C2)))) && | ||||||||
| 2245 | match(C1, m_Power2())) { | ||||||||
| 2246 | Constant *Log2C1 = ConstantExpr::getExactLogBase2(C1); | ||||||||
| 2247 | Constant *LshrC = ConstantExpr::getAdd(C2, Log2C3); | ||||||||
| 2248 | KnownBits KnownLShrc = computeKnownBits(LshrC, 0, nullptr); | ||||||||
| 2249 | if (KnownLShrc.getMaxValue().ult(Width)) { | ||||||||
| 2250 | // iff C1,C3 is pow2 and C2 + cttz(C3) < BitWidth: | ||||||||
| 2251 | // ((C1 << X) >> C2) & C3 -> X == (cttz(C3)+C2-cttz(C1)) ? C3 : 0 | ||||||||
| 2252 | Constant *CmpC = ConstantExpr::getSub(LshrC, Log2C1); | ||||||||
| 2253 | Value *Cmp = Builder.CreateICmpEQ(X, CmpC); | ||||||||
| 2254 | return SelectInst::Create(Cmp, ConstantInt::get(Ty, *C3), | ||||||||
| 2255 | ConstantInt::getNullValue(Ty)); | ||||||||
| 2256 | } | ||||||||
| 2257 | } | ||||||||
| 2258 | |||||||||
| 2259 | if (match(Op0, m_OneUse(m_Shl(m_LShr(m_ImmConstant(C1), m_Value(X)), | ||||||||
| 2260 | m_ImmConstant(C2)))) && | ||||||||
| 2261 | match(C1, m_Power2())) { | ||||||||
| 2262 | Constant *Log2C1 = ConstantExpr::getExactLogBase2(C1); | ||||||||
| 2263 | Constant *Cmp = | ||||||||
| 2264 | ConstantExpr::getCompare(ICmpInst::ICMP_ULT, Log2C3, C2); | ||||||||
| 2265 | if (Cmp->isZeroValue()) { | ||||||||
| 2266 | // iff C1,C3 is pow2 and Log2(C3) >= C2: | ||||||||
| 2267 | // ((C1 >> X) << C2) & C3 -> X == (cttz(C1)+C2-cttz(C3)) ? C3 : 0 | ||||||||
| 2268 | Constant *ShlC = ConstantExpr::getAdd(C2, Log2C1); | ||||||||
| 2269 | Constant *CmpC = ConstantExpr::getSub(ShlC, Log2C3); | ||||||||
| 2270 | Value *Cmp = Builder.CreateICmpEQ(X, CmpC); | ||||||||
| 2271 | return SelectInst::Create(Cmp, ConstantInt::get(Ty, *C3), | ||||||||
| 2272 | ConstantInt::getNullValue(Ty)); | ||||||||
| 2273 | } | ||||||||
| 2274 | } | ||||||||
| 2275 | } | ||||||||
| 2276 | } | ||||||||
| 2277 | |||||||||
| 2278 | if (match(&I, m_And(m_OneUse(m_Shl(m_ZExt(m_Value(X)), m_Value(Y))), | ||||||||
| 2279 | m_SignMask())) && | ||||||||
| 2280 | match(Y, m_SpecificInt_ICMP( | ||||||||
| 2281 | ICmpInst::Predicate::ICMP_EQ, | ||||||||
| 2282 | APInt(Ty->getScalarSizeInBits(), | ||||||||
| 2283 | Ty->getScalarSizeInBits() - | ||||||||
| 2284 | X->getType()->getScalarSizeInBits())))) { | ||||||||
| 2285 | auto *SExt = Builder.CreateSExt(X, Ty, X->getName() + ".signext"); | ||||||||
| 2286 | auto *SanitizedSignMask = cast<Constant>(Op1); | ||||||||
| 2287 | // We must be careful with the undef elements of the sign bit mask, however: | ||||||||
| 2288 | // the mask elt can be undef iff the shift amount for that lane was undef, | ||||||||
| 2289 | // otherwise we need to sanitize undef masks to zero. | ||||||||
| 2290 | SanitizedSignMask = Constant::replaceUndefsWith( | ||||||||
| 2291 | SanitizedSignMask, ConstantInt::getNullValue(Ty->getScalarType())); | ||||||||
| 2292 | SanitizedSignMask = | ||||||||
| 2293 | Constant::mergeUndefsWith(SanitizedSignMask, cast<Constant>(Y)); | ||||||||
| 2294 | return BinaryOperator::CreateAnd(SExt, SanitizedSignMask); | ||||||||
| 2295 | } | ||||||||
| 2296 | |||||||||
| 2297 | if (Instruction *Z = narrowMaskedBinOp(I)) | ||||||||
| 2298 | return Z; | ||||||||
| 2299 | |||||||||
| 2300 | if (I.getType()->isIntOrIntVectorTy(1)) { | ||||||||
| 2301 | if (auto *SI0 = dyn_cast<SelectInst>(Op0)) { | ||||||||
| 2302 | if (auto *I = | ||||||||
| 2303 | foldAndOrOfSelectUsingImpliedCond(Op1, *SI0, /* IsAnd */ true)) | ||||||||
| 2304 | return I; | ||||||||
| 2305 | } | ||||||||
| 2306 | if (auto *SI1 = dyn_cast<SelectInst>(Op1)) { | ||||||||
| 2307 | if (auto *I = | ||||||||
| 2308 | foldAndOrOfSelectUsingImpliedCond(Op0, *SI1, /* IsAnd */ true)) | ||||||||
| 2309 | return I; | ||||||||
| 2310 | } | ||||||||
| 2311 | } | ||||||||
| 2312 | |||||||||
| 2313 | if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I)) | ||||||||
| 2314 | return FoldedLogic; | ||||||||
| 2315 | |||||||||
| 2316 | if (Instruction *DeMorgan = matchDeMorgansLaws(I, Builder)) | ||||||||
| 2317 | return DeMorgan; | ||||||||
| 2318 | |||||||||
| 2319 | { | ||||||||
| 2320 | Value *A, *B, *C; | ||||||||
| 2321 | // A & (A ^ B) --> A & ~B | ||||||||
| 2322 | if (match(Op1, m_OneUse(m_c_Xor(m_Specific(Op0), m_Value(B))))) | ||||||||
| 2323 | return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(B)); | ||||||||
| 2324 | // (A ^ B) & A --> A & ~B | ||||||||
| 2325 | if (match(Op0, m_OneUse(m_c_Xor(m_Specific(Op1), m_Value(B))))) | ||||||||
| 2326 | return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(B)); | ||||||||
| 2327 | |||||||||
| 2328 | // A & ~(A ^ B) --> A & B | ||||||||
| 2329 | if (match(Op1, m_Not(m_c_Xor(m_Specific(Op0), m_Value(B))))) | ||||||||
| 2330 | return BinaryOperator::CreateAnd(Op0, B); | ||||||||
| 2331 | // ~(A ^ B) & A --> A & B | ||||||||
| 2332 | if (match(Op0, m_Not(m_c_Xor(m_Specific(Op1), m_Value(B))))) | ||||||||
| 2333 | return BinaryOperator::CreateAnd(Op1, B); | ||||||||
| 2334 | |||||||||
| 2335 | // (A ^ B) & ((B ^ C) ^ A) -> (A ^ B) & ~C | ||||||||
| 2336 | if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) | ||||||||
| 2337 | if (match(Op1, m_Xor(m_Xor(m_Specific(B), m_Value(C)), m_Specific(A)))) | ||||||||
| 2338 | if (Op1->hasOneUse() || isFreeToInvert(C, C->hasOneUse())) | ||||||||
| 2339 | return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(C)); | ||||||||
| 2340 | |||||||||
| 2341 | // ((A ^ C) ^ B) & (B ^ A) -> (B ^ A) & ~C | ||||||||
| 2342 | if (match(Op0, m_Xor(m_Xor(m_Value(A), m_Value(C)), m_Value(B)))) | ||||||||
| 2343 | if (match(Op1, m_Xor(m_Specific(B), m_Specific(A)))) | ||||||||
| 2344 | if (Op0->hasOneUse() || isFreeToInvert(C, C->hasOneUse())) | ||||||||
| 2345 | return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(C)); | ||||||||
| 2346 | |||||||||
| 2347 | // (A | B) & (~A ^ B) -> A & B | ||||||||
| 2348 | // (A | B) & (B ^ ~A) -> A & B | ||||||||
| 2349 | // (B | A) & (~A ^ B) -> A & B | ||||||||
| 2350 | // (B | A) & (B ^ ~A) -> A & B | ||||||||
| 2351 | if (match(Op1, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) && | ||||||||
| 2352 | match(Op0, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||||||
| 2353 | return BinaryOperator::CreateAnd(A, B); | ||||||||
| 2354 | |||||||||
| 2355 | // (~A ^ B) & (A | B) -> A & B | ||||||||
| 2356 | // (~A ^ B) & (B | A) -> A & B | ||||||||
| 2357 | // (B ^ ~A) & (A | B) -> A & B | ||||||||
| 2358 | // (B ^ ~A) & (B | A) -> A & B | ||||||||
| 2359 | if (match(Op0, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) && | ||||||||
| 2360 | match(Op1, m_c_Or(m_Specific(A), m_Specific(B)))) | ||||||||
| 2361 | return BinaryOperator::CreateAnd(A, B); | ||||||||
| 2362 | |||||||||
| 2363 | // (~A | B) & (A ^ B) -> ~A & B | ||||||||
| 2364 | // (~A | B) & (B ^ A) -> ~A & B | ||||||||
| 2365 | // (B | ~A) & (A ^ B) -> ~A & B | ||||||||
| 2366 | // (B | ~A) & (B ^ A) -> ~A & B | ||||||||
| 2367 | if (match(Op0, m_c_Or(m_Not(m_Value(A)), m_Value(B))) && | ||||||||
| 2368 | match(Op1, m_c_Xor(m_Specific(A), m_Specific(B)))) | ||||||||
| 2369 | return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); | ||||||||
| 2370 | |||||||||
| 2371 | // (A ^ B) & (~A | B) -> ~A & B | ||||||||
| 2372 | // (B ^ A) & (~A | B) -> ~A & B | ||||||||
| 2373 | // (A ^ B) & (B | ~A) -> ~A & B | ||||||||
| 2374 | // (B ^ A) & (B | ~A) -> ~A & B | ||||||||
| 2375 | if (match(Op1, m_c_Or(m_Not(m_Value(A)), m_Value(B))) && | ||||||||
| 2376 | match(Op0, m_c_Xor(m_Specific(A), m_Specific(B)))) | ||||||||
| 2377 | return BinaryOperator::CreateAnd(Builder.CreateNot(A), B); | ||||||||
| 2378 | } | ||||||||
| 2379 | |||||||||
| 2380 | { | ||||||||
| 2381 | ICmpInst *LHS = dyn_cast<ICmpInst>(Op0); | ||||||||
| 2382 | ICmpInst *RHS = dyn_cast<ICmpInst>(Op1); | ||||||||
| 2383 | if (LHS && RHS) | ||||||||
| 2384 | if (Value *Res = foldAndOrOfICmps(LHS, RHS, I, /* IsAnd */ true)) | ||||||||
| 2385 | return replaceInstUsesWith(I, Res); | ||||||||
| 2386 | |||||||||
| 2387 | // TODO: Make this recursive; it's a little tricky because an arbitrary | ||||||||
| 2388 | // number of 'and' instructions might have to be created. | ||||||||
| 2389 | if (LHS && match(Op1, m_OneUse(m_LogicalAnd(m_Value(X), m_Value(Y))))) { | ||||||||
| 2390 | bool IsLogical = isa<SelectInst>(Op1); | ||||||||
| 2391 | // LHS & (X && Y) --> (LHS && X) && Y | ||||||||
| 2392 | if (auto *Cmp = dyn_cast<ICmpInst>(X)) | ||||||||
| 2393 | if (Value *Res = | ||||||||
| 2394 | foldAndOrOfICmps(LHS, Cmp, I, /* IsAnd */ true, IsLogical)) | ||||||||
| 2395 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 2396 | ? Builder.CreateLogicalAnd(Res, Y) | ||||||||
| 2397 | : Builder.CreateAnd(Res, Y)); | ||||||||
| 2398 | // LHS & (X && Y) --> X && (LHS & Y) | ||||||||
| 2399 | if (auto *Cmp = dyn_cast<ICmpInst>(Y)) | ||||||||
| 2400 | if (Value *Res = foldAndOrOfICmps(LHS, Cmp, I, /* IsAnd */ true, | ||||||||
| 2401 | /* IsLogical */ false)) | ||||||||
| 2402 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 2403 | ? Builder.CreateLogicalAnd(X, Res) | ||||||||
| 2404 | : Builder.CreateAnd(X, Res)); | ||||||||
| 2405 | } | ||||||||
| 2406 | if (RHS && match(Op0, m_OneUse(m_LogicalAnd(m_Value(X), m_Value(Y))))) { | ||||||||
| 2407 | bool IsLogical = isa<SelectInst>(Op0); | ||||||||
| 2408 | // (X && Y) & RHS --> (X && RHS) && Y | ||||||||
| 2409 | if (auto *Cmp = dyn_cast<ICmpInst>(X)) | ||||||||
| 2410 | if (Value *Res = | ||||||||
| 2411 | foldAndOrOfICmps(Cmp, RHS, I, /* IsAnd */ true, IsLogical)) | ||||||||
| 2412 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 2413 | ? Builder.CreateLogicalAnd(Res, Y) | ||||||||
| 2414 | : Builder.CreateAnd(Res, Y)); | ||||||||
| 2415 | // (X && Y) & RHS --> X && (Y & RHS) | ||||||||
| 2416 | if (auto *Cmp = dyn_cast<ICmpInst>(Y)) | ||||||||
| 2417 | if (Value *Res = foldAndOrOfICmps(Cmp, RHS, I, /* IsAnd */ true, | ||||||||
| 2418 | /* IsLogical */ false)) | ||||||||
| 2419 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 2420 | ? Builder.CreateLogicalAnd(X, Res) | ||||||||
| 2421 | : Builder.CreateAnd(X, Res)); | ||||||||
| 2422 | } | ||||||||
| 2423 | } | ||||||||
| 2424 | |||||||||
| 2425 | if (FCmpInst *LHS = dyn_cast<FCmpInst>(I.getOperand(0))) | ||||||||
| 2426 | if (FCmpInst *RHS = dyn_cast<FCmpInst>(I.getOperand(1))) | ||||||||
| 2427 | if (Value *Res = foldLogicOfFCmps(LHS, RHS, /*IsAnd*/ true)) | ||||||||
| 2428 | return replaceInstUsesWith(I, Res); | ||||||||
| 2429 | |||||||||
| 2430 | if (Instruction *FoldedFCmps = reassociateFCmps(I, Builder)) | ||||||||
| 2431 | return FoldedFCmps; | ||||||||
| 2432 | |||||||||
| 2433 | if (Instruction *CastedAnd = foldCastedBitwiseLogic(I)) | ||||||||
| 2434 | return CastedAnd; | ||||||||
| 2435 | |||||||||
| 2436 | if (Instruction *Sel = foldBinopOfSextBoolToSelect(I)) | ||||||||
| 2437 | return Sel; | ||||||||
| 2438 | |||||||||
| 2439 | // and(sext(A), B) / and(B, sext(A)) --> A ? B : 0, where A is i1 or <N x i1>. | ||||||||
| 2440 | // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold | ||||||||
| 2441 | // with binop identity constant. But creating a select with non-constant | ||||||||
| 2442 | // arm may not be reversible due to poison semantics. Is that a good | ||||||||
| 2443 | // canonicalization? | ||||||||
| 2444 | Value *A; | ||||||||
| 2445 | if (match(Op0, m_OneUse(m_SExt(m_Value(A)))) && | ||||||||
| 2446 | A->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 2447 | return SelectInst::Create(A, Op1, Constant::getNullValue(Ty)); | ||||||||
| 2448 | if (match(Op1, m_OneUse(m_SExt(m_Value(A)))) && | ||||||||
| 2449 | A->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 2450 | return SelectInst::Create(A, Op0, Constant::getNullValue(Ty)); | ||||||||
| 2451 | |||||||||
| 2452 | // Similarly, a 'not' of the bool translates to a swap of the select arms: | ||||||||
| 2453 | // ~sext(A) & Op1 --> A ? 0 : Op1 | ||||||||
| 2454 | // Op0 & ~sext(A) --> A ? 0 : Op0 | ||||||||
| 2455 | if (match(Op0, m_Not(m_SExt(m_Value(A)))) && | ||||||||
| 2456 | A->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 2457 | return SelectInst::Create(A, Constant::getNullValue(Ty), Op1); | ||||||||
| 2458 | if (match(Op1, m_Not(m_SExt(m_Value(A)))) && | ||||||||
| 2459 | A->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 2460 | return SelectInst::Create(A, Constant::getNullValue(Ty), Op0); | ||||||||
| 2461 | |||||||||
| 2462 | // (iN X s>> (N-1)) & Y --> (X s< 0) ? Y : 0 -- with optional sext | ||||||||
| 2463 | if (match(&I, m_c_And(m_OneUse(m_SExtOrSelf( | ||||||||
| 2464 | m_AShr(m_Value(X), m_APIntAllowUndef(C)))), | ||||||||
| 2465 | m_Value(Y))) && | ||||||||
| 2466 | *C == X->getType()->getScalarSizeInBits() - 1) { | ||||||||
| 2467 | Value *IsNeg = Builder.CreateIsNeg(X, "isneg"); | ||||||||
| 2468 | return SelectInst::Create(IsNeg, Y, ConstantInt::getNullValue(Ty)); | ||||||||
| 2469 | } | ||||||||
| 2470 | // If there's a 'not' of the shifted value, swap the select operands: | ||||||||
| 2471 | // ~(iN X s>> (N-1)) & Y --> (X s< 0) ? 0 : Y -- with optional sext | ||||||||
| 2472 | if (match(&I, m_c_And(m_OneUse(m_SExtOrSelf( | ||||||||
| 2473 | m_Not(m_AShr(m_Value(X), m_APIntAllowUndef(C))))), | ||||||||
| 2474 | m_Value(Y))) && | ||||||||
| 2475 | *C == X->getType()->getScalarSizeInBits() - 1) { | ||||||||
| 2476 | Value *IsNeg = Builder.CreateIsNeg(X, "isneg"); | ||||||||
| 2477 | return SelectInst::Create(IsNeg, ConstantInt::getNullValue(Ty), Y); | ||||||||
| 2478 | } | ||||||||
| 2479 | |||||||||
| 2480 | // (~x) & y --> ~(x | (~y)) iff that gets rid of inversions | ||||||||
| 2481 | if (sinkNotIntoOtherHandOfLogicalOp(I)) | ||||||||
| 2482 | return &I; | ||||||||
| 2483 | |||||||||
| 2484 | // An and recurrence w/loop invariant step is equivelent to (and start, step) | ||||||||
| 2485 | PHINode *PN = nullptr; | ||||||||
| 2486 | Value *Start = nullptr, *Step = nullptr; | ||||||||
| 2487 | if (matchSimpleRecurrence(&I, PN, Start, Step) && DT.dominates(Step, PN)) | ||||||||
| 2488 | return replaceInstUsesWith(I, Builder.CreateAnd(Start, Step)); | ||||||||
| 2489 | |||||||||
| 2490 | if (Instruction *R = reassociateForUses(I, Builder)) | ||||||||
| 2491 | return R; | ||||||||
| 2492 | |||||||||
| 2493 | if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder)) | ||||||||
| 2494 | return Canonicalized; | ||||||||
| 2495 | |||||||||
| 2496 | if (Instruction *Folded = foldLogicOfIsFPClass(I, Op0, Op1)) | ||||||||
| 2497 | return Folded; | ||||||||
| 2498 | |||||||||
| 2499 | return nullptr; | ||||||||
| 2500 | } | ||||||||
| 2501 | |||||||||
| 2502 | Instruction *InstCombinerImpl::matchBSwapOrBitReverse(Instruction &I, | ||||||||
| 2503 | bool MatchBSwaps, | ||||||||
| 2504 | bool MatchBitReversals) { | ||||||||
| 2505 | SmallVector<Instruction *, 4> Insts; | ||||||||
| 2506 | if (!recognizeBSwapOrBitReverseIdiom(&I, MatchBSwaps, MatchBitReversals, | ||||||||
| 2507 | Insts)) | ||||||||
| 2508 | return nullptr; | ||||||||
| 2509 | Instruction *LastInst = Insts.pop_back_val(); | ||||||||
| 2510 | LastInst->removeFromParent(); | ||||||||
| 2511 | |||||||||
| 2512 | for (auto *Inst : Insts) | ||||||||
| 2513 | Worklist.push(Inst); | ||||||||
| 2514 | return LastInst; | ||||||||
| 2515 | } | ||||||||
| 2516 | |||||||||
| 2517 | /// Match UB-safe variants of the funnel shift intrinsic. | ||||||||
| 2518 | static Instruction *matchFunnelShift(Instruction &Or, InstCombinerImpl &IC) { | ||||||||
| 2519 | // TODO: Can we reduce the code duplication between this and the related | ||||||||
| 2520 | // rotate matching code under visitSelect and visitTrunc? | ||||||||
| 2521 | unsigned Width = Or.getType()->getScalarSizeInBits(); | ||||||||
| 2522 | |||||||||
| 2523 | // First, find an or'd pair of opposite shifts: | ||||||||
| 2524 | // or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1) | ||||||||
| 2525 | BinaryOperator *Or0, *Or1; | ||||||||
| 2526 | if (!match(Or.getOperand(0), m_BinOp(Or0)) || | ||||||||
| 2527 | !match(Or.getOperand(1), m_BinOp(Or1))) | ||||||||
| 2528 | return nullptr; | ||||||||
| 2529 | |||||||||
| 2530 | Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1; | ||||||||
| 2531 | if (!match(Or0, m_OneUse(m_LogicalShift(m_Value(ShVal0), m_Value(ShAmt0)))) || | ||||||||
| 2532 | !match(Or1, m_OneUse(m_LogicalShift(m_Value(ShVal1), m_Value(ShAmt1)))) || | ||||||||
| 2533 | Or0->getOpcode() == Or1->getOpcode()) | ||||||||
| 2534 | return nullptr; | ||||||||
| 2535 | |||||||||
| 2536 | // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)). | ||||||||
| 2537 | if (Or0->getOpcode() == BinaryOperator::LShr) { | ||||||||
| 2538 | std::swap(Or0, Or1); | ||||||||
| 2539 | std::swap(ShVal0, ShVal1); | ||||||||
| 2540 | std::swap(ShAmt0, ShAmt1); | ||||||||
| 2541 | } | ||||||||
| 2542 | assert(Or0->getOpcode() == BinaryOperator::Shl &&(static_cast <bool> (Or0->getOpcode() == BinaryOperator ::Shl && Or1->getOpcode() == BinaryOperator::LShr && "Illegal or(shift,shift) pair") ? void (0) : __assert_fail ( "Or0->getOpcode() == BinaryOperator::Shl && Or1->getOpcode() == BinaryOperator::LShr && \"Illegal or(shift,shift) pair\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 2544 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2543 | Or1->getOpcode() == BinaryOperator::LShr &&(static_cast <bool> (Or0->getOpcode() == BinaryOperator ::Shl && Or1->getOpcode() == BinaryOperator::LShr && "Illegal or(shift,shift) pair") ? void (0) : __assert_fail ( "Or0->getOpcode() == BinaryOperator::Shl && Or1->getOpcode() == BinaryOperator::LShr && \"Illegal or(shift,shift) pair\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 2544 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2544 | "Illegal or(shift,shift) pair")(static_cast <bool> (Or0->getOpcode() == BinaryOperator ::Shl && Or1->getOpcode() == BinaryOperator::LShr && "Illegal or(shift,shift) pair") ? void (0) : __assert_fail ( "Or0->getOpcode() == BinaryOperator::Shl && Or1->getOpcode() == BinaryOperator::LShr && \"Illegal or(shift,shift) pair\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 2544 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2545 | |||||||||
| 2546 | // Match the shift amount operands for a funnel shift pattern. This always | ||||||||
| 2547 | // matches a subtraction on the R operand. | ||||||||
| 2548 | auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * { | ||||||||
| 2549 | // Check for constant shift amounts that sum to the bitwidth. | ||||||||
| 2550 | const APInt *LI, *RI; | ||||||||
| 2551 | if (match(L, m_APIntAllowUndef(LI)) && match(R, m_APIntAllowUndef(RI))) | ||||||||
| 2552 | if (LI->ult(Width) && RI->ult(Width) && (*LI + *RI) == Width) | ||||||||
| 2553 | return ConstantInt::get(L->getType(), *LI); | ||||||||
| 2554 | |||||||||
| 2555 | Constant *LC, *RC; | ||||||||
| 2556 | if (match(L, m_Constant(LC)) && match(R, m_Constant(RC)) && | ||||||||
| 2557 | match(L, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) && | ||||||||
| 2558 | match(R, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) && | ||||||||
| 2559 | match(ConstantExpr::getAdd(LC, RC), m_SpecificIntAllowUndef(Width))) | ||||||||
| 2560 | return ConstantExpr::mergeUndefsWith(LC, RC); | ||||||||
| 2561 | |||||||||
| 2562 | // (shl ShVal, X) | (lshr ShVal, (Width - x)) iff X < Width. | ||||||||
| 2563 | // We limit this to X < Width in case the backend re-expands the intrinsic, | ||||||||
| 2564 | // and has to reintroduce a shift modulo operation (InstCombine might remove | ||||||||
| 2565 | // it after this fold). This still doesn't guarantee that the final codegen | ||||||||
| 2566 | // will match this original pattern. | ||||||||
| 2567 | if (match(R, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(L))))) { | ||||||||
| 2568 | KnownBits KnownL = IC.computeKnownBits(L, /*Depth*/ 0, &Or); | ||||||||
| 2569 | return KnownL.getMaxValue().ult(Width) ? L : nullptr; | ||||||||
| 2570 | } | ||||||||
| 2571 | |||||||||
| 2572 | // For non-constant cases, the following patterns currently only work for | ||||||||
| 2573 | // rotation patterns. | ||||||||
| 2574 | // TODO: Add general funnel-shift compatible patterns. | ||||||||
| 2575 | if (ShVal0 != ShVal1) | ||||||||
| 2576 | return nullptr; | ||||||||
| 2577 | |||||||||
| 2578 | // For non-constant cases we don't support non-pow2 shift masks. | ||||||||
| 2579 | // TODO: Is it worth matching urem as well? | ||||||||
| 2580 | if (!isPowerOf2_32(Width)) | ||||||||
| 2581 | return nullptr; | ||||||||
| 2582 | |||||||||
| 2583 | // The shift amount may be masked with negation: | ||||||||
| 2584 | // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1))) | ||||||||
| 2585 | Value *X; | ||||||||
| 2586 | unsigned Mask = Width - 1; | ||||||||
| 2587 | if (match(L, m_And(m_Value(X), m_SpecificInt(Mask))) && | ||||||||
| 2588 | match(R, m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask)))) | ||||||||
| 2589 | return X; | ||||||||
| 2590 | |||||||||
| 2591 | // Similar to above, but the shift amount may be extended after masking, | ||||||||
| 2592 | // so return the extended value as the parameter for the intrinsic. | ||||||||
| 2593 | if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) && | ||||||||
| 2594 | match(R, m_And(m_Neg(m_ZExt(m_And(m_Specific(X), m_SpecificInt(Mask)))), | ||||||||
| 2595 | m_SpecificInt(Mask)))) | ||||||||
| 2596 | return L; | ||||||||
| 2597 | |||||||||
| 2598 | if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) && | ||||||||
| 2599 | match(R, m_ZExt(m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask))))) | ||||||||
| 2600 | return L; | ||||||||
| 2601 | |||||||||
| 2602 | return nullptr; | ||||||||
| 2603 | }; | ||||||||
| 2604 | |||||||||
| 2605 | Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width); | ||||||||
| 2606 | bool IsFshl = true; // Sub on LSHR. | ||||||||
| 2607 | if (!ShAmt) { | ||||||||
| 2608 | ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width); | ||||||||
| 2609 | IsFshl = false; // Sub on SHL. | ||||||||
| 2610 | } | ||||||||
| 2611 | if (!ShAmt) | ||||||||
| 2612 | return nullptr; | ||||||||
| 2613 | |||||||||
| 2614 | Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr; | ||||||||
| 2615 | Function *F = Intrinsic::getDeclaration(Or.getModule(), IID, Or.getType()); | ||||||||
| 2616 | return CallInst::Create(F, {ShVal0, ShVal1, ShAmt}); | ||||||||
| 2617 | } | ||||||||
| 2618 | |||||||||
| 2619 | /// Attempt to combine or(zext(x),shl(zext(y),bw/2) concat packing patterns. | ||||||||
| 2620 | static Instruction *matchOrConcat(Instruction &Or, | ||||||||
| 2621 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 2622 | assert(Or.getOpcode() == Instruction::Or && "bswap requires an 'or'")(static_cast <bool> (Or.getOpcode() == Instruction::Or && "bswap requires an 'or'") ? void (0) : __assert_fail ("Or.getOpcode() == Instruction::Or && \"bswap requires an 'or'\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 2622 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2623 | Value *Op0 = Or.getOperand(0), *Op1 = Or.getOperand(1); | ||||||||
| 2624 | Type *Ty = Or.getType(); | ||||||||
| 2625 | |||||||||
| 2626 | unsigned Width = Ty->getScalarSizeInBits(); | ||||||||
| 2627 | if ((Width & 1) != 0) | ||||||||
| 2628 | return nullptr; | ||||||||
| 2629 | unsigned HalfWidth = Width / 2; | ||||||||
| 2630 | |||||||||
| 2631 | // Canonicalize zext (lower half) to LHS. | ||||||||
| 2632 | if (!isa<ZExtInst>(Op0)) | ||||||||
| 2633 | std::swap(Op0, Op1); | ||||||||
| 2634 | |||||||||
| 2635 | // Find lower/upper half. | ||||||||
| 2636 | Value *LowerSrc, *ShlVal, *UpperSrc; | ||||||||
| 2637 | const APInt *C; | ||||||||
| 2638 | if (!match(Op0, m_OneUse(m_ZExt(m_Value(LowerSrc)))) || | ||||||||
| 2639 | !match(Op1, m_OneUse(m_Shl(m_Value(ShlVal), m_APInt(C)))) || | ||||||||
| 2640 | !match(ShlVal, m_OneUse(m_ZExt(m_Value(UpperSrc))))) | ||||||||
| 2641 | return nullptr; | ||||||||
| 2642 | if (*C != HalfWidth || LowerSrc->getType() != UpperSrc->getType() || | ||||||||
| 2643 | LowerSrc->getType()->getScalarSizeInBits() != HalfWidth) | ||||||||
| 2644 | return nullptr; | ||||||||
| 2645 | |||||||||
| 2646 | auto ConcatIntrinsicCalls = [&](Intrinsic::ID id, Value *Lo, Value *Hi) { | ||||||||
| 2647 | Value *NewLower = Builder.CreateZExt(Lo, Ty); | ||||||||
| 2648 | Value *NewUpper = Builder.CreateZExt(Hi, Ty); | ||||||||
| 2649 | NewUpper = Builder.CreateShl(NewUpper, HalfWidth); | ||||||||
| 2650 | Value *BinOp = Builder.CreateOr(NewLower, NewUpper); | ||||||||
| 2651 | Function *F = Intrinsic::getDeclaration(Or.getModule(), id, Ty); | ||||||||
| 2652 | return Builder.CreateCall(F, BinOp); | ||||||||
| 2653 | }; | ||||||||
| 2654 | |||||||||
| 2655 | // BSWAP: Push the concat down, swapping the lower/upper sources. | ||||||||
| 2656 | // concat(bswap(x),bswap(y)) -> bswap(concat(x,y)) | ||||||||
| 2657 | Value *LowerBSwap, *UpperBSwap; | ||||||||
| 2658 | if (match(LowerSrc, m_BSwap(m_Value(LowerBSwap))) && | ||||||||
| 2659 | match(UpperSrc, m_BSwap(m_Value(UpperBSwap)))) | ||||||||
| 2660 | return ConcatIntrinsicCalls(Intrinsic::bswap, UpperBSwap, LowerBSwap); | ||||||||
| 2661 | |||||||||
| 2662 | // BITREVERSE: Push the concat down, swapping the lower/upper sources. | ||||||||
| 2663 | // concat(bitreverse(x),bitreverse(y)) -> bitreverse(concat(x,y)) | ||||||||
| 2664 | Value *LowerBRev, *UpperBRev; | ||||||||
| 2665 | if (match(LowerSrc, m_BitReverse(m_Value(LowerBRev))) && | ||||||||
| 2666 | match(UpperSrc, m_BitReverse(m_Value(UpperBRev)))) | ||||||||
| 2667 | return ConcatIntrinsicCalls(Intrinsic::bitreverse, UpperBRev, LowerBRev); | ||||||||
| 2668 | |||||||||
| 2669 | return nullptr; | ||||||||
| 2670 | } | ||||||||
| 2671 | |||||||||
| 2672 | /// If all elements of two constant vectors are 0/-1 and inverses, return true. | ||||||||
| 2673 | static bool areInverseVectorBitmasks(Constant *C1, Constant *C2) { | ||||||||
| 2674 | unsigned NumElts = cast<FixedVectorType>(C1->getType())->getNumElements(); | ||||||||
| 2675 | for (unsigned i = 0; i != NumElts; ++i) { | ||||||||
| 2676 | Constant *EltC1 = C1->getAggregateElement(i); | ||||||||
| 2677 | Constant *EltC2 = C2->getAggregateElement(i); | ||||||||
| 2678 | if (!EltC1 || !EltC2) | ||||||||
| 2679 | return false; | ||||||||
| 2680 | |||||||||
| 2681 | // One element must be all ones, and the other must be all zeros. | ||||||||
| 2682 | if (!((match(EltC1, m_Zero()) && match(EltC2, m_AllOnes())) || | ||||||||
| 2683 | (match(EltC2, m_Zero()) && match(EltC1, m_AllOnes())))) | ||||||||
| 2684 | return false; | ||||||||
| 2685 | } | ||||||||
| 2686 | return true; | ||||||||
| 2687 | } | ||||||||
| 2688 | |||||||||
| 2689 | /// We have an expression of the form (A & C) | (B & D). If A is a scalar or | ||||||||
| 2690 | /// vector composed of all-zeros or all-ones values and is the bitwise 'not' of | ||||||||
| 2691 | /// B, it can be used as the condition operand of a select instruction. | ||||||||
| 2692 | /// We will detect (A & C) | ~(B | D) when the flag ABIsTheSame enabled. | ||||||||
| 2693 | Value *InstCombinerImpl::getSelectCondition(Value *A, Value *B, | ||||||||
| 2694 | bool ABIsTheSame) { | ||||||||
| 2695 | // We may have peeked through bitcasts in the caller. | ||||||||
| 2696 | // Exit immediately if we don't have (vector) integer types. | ||||||||
| 2697 | Type *Ty = A->getType(); | ||||||||
| 2698 | if (!Ty->isIntOrIntVectorTy() || !B->getType()->isIntOrIntVectorTy()) | ||||||||
| 2699 | return nullptr; | ||||||||
| 2700 | |||||||||
| 2701 | // If A is the 'not' operand of B and has enough signbits, we have our answer. | ||||||||
| 2702 | if (ABIsTheSame ? (A == B) : match(B, m_Not(m_Specific(A)))) { | ||||||||
| 2703 | // If these are scalars or vectors of i1, A can be used directly. | ||||||||
| 2704 | if (Ty->isIntOrIntVectorTy(1)) | ||||||||
| 2705 | return A; | ||||||||
| 2706 | |||||||||
| 2707 | // If we look through a vector bitcast, the caller will bitcast the operands | ||||||||
| 2708 | // to match the condition's number of bits (N x i1). | ||||||||
| 2709 | // To make this poison-safe, disallow bitcast from wide element to narrow | ||||||||
| 2710 | // element. That could allow poison in lanes where it was not present in the | ||||||||
| 2711 | // original code. | ||||||||
| 2712 | A = peekThroughBitcast(A); | ||||||||
| 2713 | if (A->getType()->isIntOrIntVectorTy()) { | ||||||||
| 2714 | unsigned NumSignBits = ComputeNumSignBits(A); | ||||||||
| 2715 | if (NumSignBits == A->getType()->getScalarSizeInBits() && | ||||||||
| 2716 | NumSignBits <= Ty->getScalarSizeInBits()) | ||||||||
| 2717 | return Builder.CreateTrunc(A, CmpInst::makeCmpResultType(A->getType())); | ||||||||
| 2718 | } | ||||||||
| 2719 | return nullptr; | ||||||||
| 2720 | } | ||||||||
| 2721 | |||||||||
| 2722 | // TODO: add support for sext and constant case | ||||||||
| 2723 | if (ABIsTheSame) | ||||||||
| 2724 | return nullptr; | ||||||||
| 2725 | |||||||||
| 2726 | // If both operands are constants, see if the constants are inverse bitmasks. | ||||||||
| 2727 | Constant *AConst, *BConst; | ||||||||
| 2728 | if (match(A, m_Constant(AConst)) && match(B, m_Constant(BConst))) | ||||||||
| 2729 | if (AConst == ConstantExpr::getNot(BConst) && | ||||||||
| 2730 | ComputeNumSignBits(A) == Ty->getScalarSizeInBits()) | ||||||||
| 2731 | return Builder.CreateZExtOrTrunc(A, CmpInst::makeCmpResultType(Ty)); | ||||||||
| 2732 | |||||||||
| 2733 | // Look for more complex patterns. The 'not' op may be hidden behind various | ||||||||
| 2734 | // casts. Look through sexts and bitcasts to find the booleans. | ||||||||
| 2735 | Value *Cond; | ||||||||
| 2736 | Value *NotB; | ||||||||
| 2737 | if (match(A, m_SExt(m_Value(Cond))) && | ||||||||
| 2738 | Cond->getType()->isIntOrIntVectorTy(1)) { | ||||||||
| 2739 | // A = sext i1 Cond; B = sext (not (i1 Cond)) | ||||||||
| 2740 | if (match(B, m_SExt(m_Not(m_Specific(Cond))))) | ||||||||
| 2741 | return Cond; | ||||||||
| 2742 | |||||||||
| 2743 | // A = sext i1 Cond; B = not ({bitcast} (sext (i1 Cond))) | ||||||||
| 2744 | // TODO: The one-use checks are unnecessary or misplaced. If the caller | ||||||||
| 2745 | // checked for uses on logic ops/casts, that should be enough to | ||||||||
| 2746 | // make this transform worthwhile. | ||||||||
| 2747 | if (match(B, m_OneUse(m_Not(m_Value(NotB))))) { | ||||||||
| 2748 | NotB = peekThroughBitcast(NotB, true); | ||||||||
| 2749 | if (match(NotB, m_SExt(m_Specific(Cond)))) | ||||||||
| 2750 | return Cond; | ||||||||
| 2751 | } | ||||||||
| 2752 | } | ||||||||
| 2753 | |||||||||
| 2754 | // All scalar (and most vector) possibilities should be handled now. | ||||||||
| 2755 | // Try more matches that only apply to non-splat constant vectors. | ||||||||
| 2756 | if (!Ty->isVectorTy()) | ||||||||
| 2757 | return nullptr; | ||||||||
| 2758 | |||||||||
| 2759 | // If both operands are xor'd with constants using the same sexted boolean | ||||||||
| 2760 | // operand, see if the constants are inverse bitmasks. | ||||||||
| 2761 | // TODO: Use ConstantExpr::getNot()? | ||||||||
| 2762 | if (match(A, (m_Xor(m_SExt(m_Value(Cond)), m_Constant(AConst)))) && | ||||||||
| 2763 | match(B, (m_Xor(m_SExt(m_Specific(Cond)), m_Constant(BConst)))) && | ||||||||
| 2764 | Cond->getType()->isIntOrIntVectorTy(1) && | ||||||||
| 2765 | areInverseVectorBitmasks(AConst, BConst)) { | ||||||||
| 2766 | AConst = ConstantExpr::getTrunc(AConst, CmpInst::makeCmpResultType(Ty)); | ||||||||
| 2767 | return Builder.CreateXor(Cond, AConst); | ||||||||
| 2768 | } | ||||||||
| 2769 | return nullptr; | ||||||||
| 2770 | } | ||||||||
| 2771 | |||||||||
| 2772 | /// We have an expression of the form (A & C) | (B & D). Try to simplify this | ||||||||
| 2773 | /// to "A' ? C : D", where A' is a boolean or vector of booleans. | ||||||||
| 2774 | /// When InvertFalseVal is set to true, we try to match the pattern | ||||||||
| 2775 | /// where we have peeked through a 'not' op and A and B are the same: | ||||||||
| 2776 | /// (A & C) | ~(A | D) --> (A & C) | (~A & ~D) --> A' ? C : ~D | ||||||||
| 2777 | Value *InstCombinerImpl::matchSelectFromAndOr(Value *A, Value *C, Value *B, | ||||||||
| 2778 | Value *D, bool InvertFalseVal) { | ||||||||
| 2779 | // The potential condition of the select may be bitcasted. In that case, look | ||||||||
| 2780 | // through its bitcast and the corresponding bitcast of the 'not' condition. | ||||||||
| 2781 | Type *OrigType = A->getType(); | ||||||||
| 2782 | A = peekThroughBitcast(A, true); | ||||||||
| 2783 | B = peekThroughBitcast(B, true); | ||||||||
| 2784 | if (Value *Cond = getSelectCondition(A, B, InvertFalseVal)) { | ||||||||
| 2785 | // ((bc Cond) & C) | ((bc ~Cond) & D) --> bc (select Cond, (bc C), (bc D)) | ||||||||
| 2786 | // If this is a vector, we may need to cast to match the condition's length. | ||||||||
| 2787 | // The bitcasts will either all exist or all not exist. The builder will | ||||||||
| 2788 | // not create unnecessary casts if the types already match. | ||||||||
| 2789 | Type *SelTy = A->getType(); | ||||||||
| 2790 | if (auto *VecTy = dyn_cast<VectorType>(Cond->getType())) { | ||||||||
| 2791 | // For a fixed or scalable vector get N from <{vscale x} N x iM> | ||||||||
| 2792 | unsigned Elts = VecTy->getElementCount().getKnownMinValue(); | ||||||||
| 2793 | // For a fixed or scalable vector, get the size in bits of N x iM; for a | ||||||||
| 2794 | // scalar this is just M. | ||||||||
| 2795 | unsigned SelEltSize = SelTy->getPrimitiveSizeInBits().getKnownMinValue(); | ||||||||
| 2796 | Type *EltTy = Builder.getIntNTy(SelEltSize / Elts); | ||||||||
| 2797 | SelTy = VectorType::get(EltTy, VecTy->getElementCount()); | ||||||||
| 2798 | } | ||||||||
| 2799 | Value *BitcastC = Builder.CreateBitCast(C, SelTy); | ||||||||
| 2800 | if (InvertFalseVal) | ||||||||
| 2801 | D = Builder.CreateNot(D); | ||||||||
| 2802 | Value *BitcastD = Builder.CreateBitCast(D, SelTy); | ||||||||
| 2803 | Value *Select = Builder.CreateSelect(Cond, BitcastC, BitcastD); | ||||||||
| 2804 | return Builder.CreateBitCast(Select, OrigType); | ||||||||
| 2805 | } | ||||||||
| 2806 | |||||||||
| 2807 | return nullptr; | ||||||||
| 2808 | } | ||||||||
| 2809 | |||||||||
| 2810 | // (icmp eq X, 0) | (icmp ult Other, X) -> (icmp ule Other, X-1) | ||||||||
| 2811 | // (icmp ne X, 0) & (icmp uge Other, X) -> (icmp ugt Other, X-1) | ||||||||
| 2812 | static Value *foldAndOrOfICmpEqZeroAndICmp(ICmpInst *LHS, ICmpInst *RHS, | ||||||||
| 2813 | bool IsAnd, bool IsLogical, | ||||||||
| 2814 | IRBuilderBase &Builder) { | ||||||||
| 2815 | ICmpInst::Predicate LPred = | ||||||||
| 2816 | IsAnd ? LHS->getInversePredicate() : LHS->getPredicate(); | ||||||||
| 2817 | ICmpInst::Predicate RPred = | ||||||||
| 2818 | IsAnd ? RHS->getInversePredicate() : RHS->getPredicate(); | ||||||||
| 2819 | Value *LHS0 = LHS->getOperand(0); | ||||||||
| 2820 | if (LPred != ICmpInst::ICMP_EQ || !match(LHS->getOperand(1), m_Zero()) || | ||||||||
| 2821 | !LHS0->getType()->isIntOrIntVectorTy() || | ||||||||
| 2822 | !(LHS->hasOneUse() || RHS->hasOneUse())) | ||||||||
| 2823 | return nullptr; | ||||||||
| 2824 | |||||||||
| 2825 | Value *Other; | ||||||||
| 2826 | if (RPred == ICmpInst::ICMP_ULT && RHS->getOperand(1) == LHS0) | ||||||||
| 2827 | Other = RHS->getOperand(0); | ||||||||
| 2828 | else if (RPred == ICmpInst::ICMP_UGT && RHS->getOperand(0) == LHS0) | ||||||||
| 2829 | Other = RHS->getOperand(1); | ||||||||
| 2830 | else | ||||||||
| 2831 | return nullptr; | ||||||||
| 2832 | |||||||||
| 2833 | if (IsLogical) | ||||||||
| 2834 | Other = Builder.CreateFreeze(Other); | ||||||||
| 2835 | return Builder.CreateICmp( | ||||||||
| 2836 | IsAnd ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE, | ||||||||
| 2837 | Builder.CreateAdd(LHS0, Constant::getAllOnesValue(LHS0->getType())), | ||||||||
| 2838 | Other); | ||||||||
| 2839 | } | ||||||||
| 2840 | |||||||||
| 2841 | /// Fold (icmp)&(icmp) or (icmp)|(icmp) if possible. | ||||||||
| 2842 | /// If IsLogical is true, then the and/or is in select form and the transform | ||||||||
| 2843 | /// must be poison-safe. | ||||||||
| 2844 | Value *InstCombinerImpl::foldAndOrOfICmps(ICmpInst *LHS, ICmpInst *RHS, | ||||||||
| 2845 | Instruction &I, bool IsAnd, | ||||||||
| 2846 | bool IsLogical) { | ||||||||
| 2847 | const SimplifyQuery Q = SQ.getWithInstruction(&I); | ||||||||
| 2848 | |||||||||
| 2849 | // Fold (iszero(A & K1) | iszero(A & K2)) -> (A & (K1 | K2)) != (K1 | K2) | ||||||||
| 2850 | // Fold (!iszero(A & K1) & !iszero(A & K2)) -> (A & (K1 | K2)) == (K1 | K2) | ||||||||
| 2851 | // if K1 and K2 are a one-bit mask. | ||||||||
| 2852 | if (Value *V
| ||||||||
| |||||||||
| 2853 | return V; | ||||||||
| 2854 | |||||||||
| 2855 | ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate(); | ||||||||
| 2856 | Value *LHS0 = LHS->getOperand(0), *RHS0 = RHS->getOperand(0); | ||||||||
| 2857 | Value *LHS1 = LHS->getOperand(1), *RHS1 = RHS->getOperand(1); | ||||||||
| 2858 | const APInt *LHSC = nullptr, *RHSC = nullptr; | ||||||||
| 2859 | match(LHS1, m_APInt(LHSC)); | ||||||||
| 2860 | match(RHS1, m_APInt(RHSC)); | ||||||||
| 2861 | |||||||||
| 2862 | // (icmp1 A, B) | (icmp2 A, B) --> (icmp3 A, B) | ||||||||
| 2863 | // (icmp1 A, B) & (icmp2 A, B) --> (icmp3 A, B) | ||||||||
| 2864 | if (predicatesFoldable(PredL, PredR)) { | ||||||||
| 2865 | if (LHS0 == RHS1 && LHS1 == RHS0) { | ||||||||
| 2866 | PredL = ICmpInst::getSwappedPredicate(PredL); | ||||||||
| 2867 | std::swap(LHS0, LHS1); | ||||||||
| 2868 | } | ||||||||
| 2869 | if (LHS0 == RHS0 && LHS1 == RHS1) { | ||||||||
| 2870 | unsigned Code = IsAnd ? getICmpCode(PredL) & getICmpCode(PredR) | ||||||||
| 2871 | : getICmpCode(PredL) | getICmpCode(PredR); | ||||||||
| 2872 | bool IsSigned = LHS->isSigned() || RHS->isSigned(); | ||||||||
| 2873 | return getNewICmpValue(Code, IsSigned, LHS0, LHS1, Builder); | ||||||||
| 2874 | } | ||||||||
| 2875 | } | ||||||||
| 2876 | |||||||||
| 2877 | // handle (roughly): | ||||||||
| 2878 | // (icmp ne (A & B), C) | (icmp ne (A & D), E) | ||||||||
| 2879 | // (icmp eq (A & B), C) & (icmp eq (A & D), E) | ||||||||
| 2880 | if (Value *V = foldLogOpOfMaskedICmps(LHS, RHS, IsAnd, IsLogical, Builder)) | ||||||||
| 2881 | return V; | ||||||||
| 2882 | |||||||||
| 2883 | if (Value *V = | ||||||||
| 2884 | foldAndOrOfICmpEqZeroAndICmp(LHS, RHS, IsAnd, IsLogical, Builder)) | ||||||||
| 2885 | return V; | ||||||||
| 2886 | // We can treat logical like bitwise here, because both operands are used on | ||||||||
| 2887 | // the LHS, and as such poison from both will propagate. | ||||||||
| 2888 | if (Value *V = foldAndOrOfICmpEqZeroAndICmp(RHS, LHS, IsAnd, | ||||||||
| 2889 | /*IsLogical*/ false, Builder)) | ||||||||
| 2890 | return V; | ||||||||
| 2891 | |||||||||
| 2892 | if (Value *V = | ||||||||
| 2893 | foldAndOrOfICmpsWithConstEq(LHS, RHS, IsAnd, IsLogical, Builder, Q)) | ||||||||
| 2894 | return V; | ||||||||
| 2895 | // We can convert this case to bitwise and, because both operands are used | ||||||||
| 2896 | // on the LHS, and as such poison from both will propagate. | ||||||||
| 2897 | if (Value *V = foldAndOrOfICmpsWithConstEq(RHS, LHS, IsAnd, | ||||||||
| 2898 | /*IsLogical*/ false, Builder, Q)) | ||||||||
| 2899 | return V; | ||||||||
| 2900 | |||||||||
| 2901 | if (Value *V = foldIsPowerOf2OrZero(LHS, RHS, IsAnd, Builder)) | ||||||||
| 2902 | return V; | ||||||||
| 2903 | if (Value *V = foldIsPowerOf2OrZero(RHS, LHS, IsAnd, Builder)) | ||||||||
| 2904 | return V; | ||||||||
| 2905 | |||||||||
| 2906 | // TODO: One of these directions is fine with logical and/or, the other could | ||||||||
| 2907 | // be supported by inserting freeze. | ||||||||
| 2908 | if (!IsLogical) { | ||||||||
| 2909 | // E.g. (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n | ||||||||
| 2910 | // E.g. (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n | ||||||||
| 2911 | if (Value *V = simplifyRangeCheck(LHS, RHS, /*Inverted=*/!IsAnd)) | ||||||||
| 2912 | return V; | ||||||||
| 2913 | |||||||||
| 2914 | // E.g. (icmp sgt x, n) | (icmp slt x, 0) --> icmp ugt x, n | ||||||||
| 2915 | // E.g. (icmp slt x, n) & (icmp sge x, 0) --> icmp ult x, n | ||||||||
| 2916 | if (Value *V = simplifyRangeCheck(RHS, LHS, /*Inverted=*/!IsAnd)) | ||||||||
| 2917 | return V; | ||||||||
| 2918 | } | ||||||||
| 2919 | |||||||||
| 2920 | // TODO: Add conjugated or fold, check whether it is safe for logical and/or. | ||||||||
| 2921 | if (IsAnd && !IsLogical) | ||||||||
| 2922 | if (Value *V = foldSignedTruncationCheck(LHS, RHS, I, Builder)) | ||||||||
| 2923 | return V; | ||||||||
| 2924 | |||||||||
| 2925 | if (Value *V = foldIsPowerOf2(LHS, RHS, IsAnd, Builder)) | ||||||||
| 2926 | return V; | ||||||||
| 2927 | |||||||||
| 2928 | // TODO: Verify whether this is safe for logical and/or. | ||||||||
| 2929 | if (!IsLogical) { | ||||||||
| 2930 | if (Value *X = foldUnsignedUnderflowCheck(LHS, RHS, IsAnd, Q, Builder)) | ||||||||
| 2931 | return X; | ||||||||
| 2932 | if (Value *X = foldUnsignedUnderflowCheck(RHS, LHS, IsAnd, Q, Builder)) | ||||||||
| 2933 | return X; | ||||||||
| 2934 | } | ||||||||
| 2935 | |||||||||
| 2936 | if (Value *X = foldEqOfParts(LHS, RHS, IsAnd)) | ||||||||
| 2937 | return X; | ||||||||
| 2938 | |||||||||
| 2939 | // (icmp ne A, 0) | (icmp ne B, 0) --> (icmp ne (A|B), 0) | ||||||||
| 2940 | // (icmp eq A, 0) & (icmp eq B, 0) --> (icmp eq (A|B), 0) | ||||||||
| 2941 | // TODO: Remove this when foldLogOpOfMaskedICmps can handle undefs. | ||||||||
| 2942 | if (!IsLogical && PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) && | ||||||||
| 2943 | PredL == PredR && match(LHS1, m_ZeroInt()) && match(RHS1, m_ZeroInt()) && | ||||||||
| 2944 | LHS0->getType() == RHS0->getType()) { | ||||||||
| 2945 | Value *NewOr = Builder.CreateOr(LHS0, RHS0); | ||||||||
| 2946 | return Builder.CreateICmp(PredL, NewOr, | ||||||||
| 2947 | Constant::getNullValue(NewOr->getType())); | ||||||||
| 2948 | } | ||||||||
| 2949 | |||||||||
| 2950 | // This only handles icmp of constants: (icmp1 A, C1) | (icmp2 B, C2). | ||||||||
| 2951 | if (!LHSC || !RHSC) | ||||||||
| 2952 | return nullptr; | ||||||||
| 2953 | |||||||||
| 2954 | // (trunc x) == C1 & (and x, CA) == C2 -> (and x, CA|CMAX) == C1|C2 | ||||||||
| 2955 | // (trunc x) != C1 | (and x, CA) != C2 -> (and x, CA|CMAX) != C1|C2 | ||||||||
| 2956 | // where CMAX is the all ones value for the truncated type, | ||||||||
| 2957 | // iff the lower bits of C2 and CA are zero. | ||||||||
| 2958 | if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) && | ||||||||
| 2959 | PredL == PredR && LHS->hasOneUse() && RHS->hasOneUse()) { | ||||||||
| 2960 | Value *V; | ||||||||
| 2961 | const APInt *AndC, *SmallC = nullptr, *BigC = nullptr; | ||||||||
| 2962 | |||||||||
| 2963 | // (trunc x) == C1 & (and x, CA) == C2 | ||||||||
| 2964 | // (and x, CA) == C2 & (trunc x) == C1 | ||||||||
| 2965 | if (match(RHS0, m_Trunc(m_Value(V))) && | ||||||||
| 2966 | match(LHS0, m_And(m_Specific(V), m_APInt(AndC)))) { | ||||||||
| 2967 | SmallC = RHSC; | ||||||||
| 2968 | BigC = LHSC; | ||||||||
| 2969 | } else if (match(LHS0, m_Trunc(m_Value(V))) && | ||||||||
| 2970 | match(RHS0, m_And(m_Specific(V), m_APInt(AndC)))) { | ||||||||
| 2971 | SmallC = LHSC; | ||||||||
| 2972 | BigC = RHSC; | ||||||||
| 2973 | } | ||||||||
| 2974 | |||||||||
| 2975 | if (SmallC && BigC) { | ||||||||
| 2976 | unsigned BigBitSize = BigC->getBitWidth(); | ||||||||
| 2977 | unsigned SmallBitSize = SmallC->getBitWidth(); | ||||||||
| 2978 | |||||||||
| 2979 | // Check that the low bits are zero. | ||||||||
| 2980 | APInt Low = APInt::getLowBitsSet(BigBitSize, SmallBitSize); | ||||||||
| 2981 | if ((Low & *AndC).isZero() && (Low & *BigC).isZero()) { | ||||||||
| 2982 | Value *NewAnd = Builder.CreateAnd(V, Low | *AndC); | ||||||||
| 2983 | APInt N = SmallC->zext(BigBitSize) | *BigC; | ||||||||
| 2984 | Value *NewVal = ConstantInt::get(NewAnd->getType(), N); | ||||||||
| 2985 | return Builder.CreateICmp(PredL, NewAnd, NewVal); | ||||||||
| 2986 | } | ||||||||
| 2987 | } | ||||||||
| 2988 | } | ||||||||
| 2989 | |||||||||
| 2990 | // Match naive pattern (and its inverted form) for checking if two values | ||||||||
| 2991 | // share same sign. An example of the pattern: | ||||||||
| 2992 | // (icmp slt (X & Y), 0) | (icmp sgt (X | Y), -1) -> (icmp sgt (X ^ Y), -1) | ||||||||
| 2993 | // Inverted form (example): | ||||||||
| 2994 | // (icmp slt (X | Y), 0) & (icmp sgt (X & Y), -1) -> (icmp slt (X ^ Y), 0) | ||||||||
| 2995 | bool TrueIfSignedL, TrueIfSignedR; | ||||||||
| 2996 | if (isSignBitCheck(PredL, *LHSC, TrueIfSignedL) && | ||||||||
| 2997 | isSignBitCheck(PredR, *RHSC, TrueIfSignedR) && | ||||||||
| 2998 | (RHS->hasOneUse() || LHS->hasOneUse())) { | ||||||||
| 2999 | Value *X, *Y; | ||||||||
| 3000 | if (IsAnd) { | ||||||||
| 3001 | if ((TrueIfSignedL && !TrueIfSignedR && | ||||||||
| 3002 | match(LHS0, m_Or(m_Value(X), m_Value(Y))) && | ||||||||
| 3003 | match(RHS0, m_c_And(m_Specific(X), m_Specific(Y)))) || | ||||||||
| 3004 | (!TrueIfSignedL && TrueIfSignedR && | ||||||||
| 3005 | match(LHS0, m_And(m_Value(X), m_Value(Y))) && | ||||||||
| 3006 | match(RHS0, m_c_Or(m_Specific(X), m_Specific(Y))))) { | ||||||||
| 3007 | Value *NewXor = Builder.CreateXor(X, Y); | ||||||||
| 3008 | return Builder.CreateIsNeg(NewXor); | ||||||||
| 3009 | } | ||||||||
| 3010 | } else { | ||||||||
| 3011 | if ((TrueIfSignedL && !TrueIfSignedR && | ||||||||
| 3012 | match(LHS0, m_And(m_Value(X), m_Value(Y))) && | ||||||||
| 3013 | match(RHS0, m_c_Or(m_Specific(X), m_Specific(Y)))) || | ||||||||
| 3014 | (!TrueIfSignedL && TrueIfSignedR && | ||||||||
| 3015 | match(LHS0, m_Or(m_Value(X), m_Value(Y))) && | ||||||||
| 3016 | match(RHS0, m_c_And(m_Specific(X), m_Specific(Y))))) { | ||||||||
| 3017 | Value *NewXor = Builder.CreateXor(X, Y); | ||||||||
| 3018 | return Builder.CreateIsNotNeg(NewXor); | ||||||||
| 3019 | } | ||||||||
| 3020 | } | ||||||||
| 3021 | } | ||||||||
| 3022 | |||||||||
| 3023 | return foldAndOrOfICmpsUsingRanges(LHS, RHS, IsAnd); | ||||||||
| 3024 | } | ||||||||
| 3025 | |||||||||
| 3026 | // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches | ||||||||
| 3027 | // here. We should standardize that construct where it is needed or choose some | ||||||||
| 3028 | // other way to ensure that commutated variants of patterns are not missed. | ||||||||
| 3029 | Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) { | ||||||||
| 3030 | if (Value *V = simplifyOrInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 3031 | SQ.getWithInstruction(&I))) | ||||||||
| 3032 | return replaceInstUsesWith(I, V); | ||||||||
| 3033 | |||||||||
| 3034 | if (SimplifyAssociativeOrCommutative(I)) | ||||||||
| 3035 | return &I; | ||||||||
| 3036 | |||||||||
| 3037 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 3038 | return X; | ||||||||
| 3039 | |||||||||
| 3040 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | ||||||||
| 3041 | return Phi; | ||||||||
| 3042 | |||||||||
| 3043 | // See if we can simplify any instructions used by the instruction whose sole | ||||||||
| 3044 | // purpose is to compute bits we don't care about. | ||||||||
| 3045 | if (SimplifyDemandedInstructionBits(I)) | ||||||||
| 3046 | return &I; | ||||||||
| 3047 | |||||||||
| 3048 | // Do this before using distributive laws to catch simple and/or/not patterns. | ||||||||
| 3049 | if (Instruction *Xor = foldOrToXor(I, Builder)) | ||||||||
| 3050 | return Xor; | ||||||||
| 3051 | |||||||||
| 3052 | if (Instruction *X = foldComplexAndOrPatterns(I, Builder)) | ||||||||
| 3053 | return X; | ||||||||
| 3054 | |||||||||
| 3055 | // (A&B)|(A&C) -> A&(B|C) etc | ||||||||
| 3056 | if (Value *V = foldUsingDistributiveLaws(I)) | ||||||||
| 3057 | return replaceInstUsesWith(I, V); | ||||||||
| 3058 | |||||||||
| 3059 | if (Value *V = SimplifyBSwap(I, Builder)) | ||||||||
| 3060 | return replaceInstUsesWith(I, V); | ||||||||
| 3061 | |||||||||
| 3062 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 3063 | Type *Ty = I.getType(); | ||||||||
| 3064 | if (Ty->isIntOrIntVectorTy(1)) { | ||||||||
| 3065 | if (auto *SI0 = dyn_cast<SelectInst>(Op0)) { | ||||||||
| 3066 | if (auto *I = | ||||||||
| 3067 | foldAndOrOfSelectUsingImpliedCond(Op1, *SI0, /* IsAnd */ false)) | ||||||||
| 3068 | return I; | ||||||||
| 3069 | } | ||||||||
| 3070 | if (auto *SI1 = dyn_cast<SelectInst>(Op1)) { | ||||||||
| 3071 | if (auto *I = | ||||||||
| 3072 | foldAndOrOfSelectUsingImpliedCond(Op0, *SI1, /* IsAnd */ false)) | ||||||||
| 3073 | return I; | ||||||||
| 3074 | } | ||||||||
| 3075 | } | ||||||||
| 3076 | |||||||||
| 3077 | if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I)) | ||||||||
| 3078 | return FoldedLogic; | ||||||||
| 3079 | |||||||||
| 3080 | if (Instruction *BitOp = matchBSwapOrBitReverse(I, /*MatchBSwaps*/ true, | ||||||||
| 3081 | /*MatchBitReversals*/ true)) | ||||||||
| 3082 | return BitOp; | ||||||||
| 3083 | |||||||||
| 3084 | if (Instruction *Funnel = matchFunnelShift(I, *this)) | ||||||||
| 3085 | return Funnel; | ||||||||
| 3086 | |||||||||
| 3087 | if (Instruction *Concat = matchOrConcat(I, Builder)) | ||||||||
| 3088 | return replaceInstUsesWith(I, Concat); | ||||||||
| 3089 | |||||||||
| 3090 | Value *X, *Y; | ||||||||
| 3091 | const APInt *CV; | ||||||||
| 3092 | if (match(&I, m_c_Or(m_OneUse(m_Xor(m_Value(X), m_APInt(CV))), m_Value(Y))) && | ||||||||
| 3093 | !CV->isAllOnes() && MaskedValueIsZero(Y, *CV, 0, &I)) { | ||||||||
| 3094 | // (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0 | ||||||||
| 3095 | // The check for a 'not' op is for efficiency (if Y is known zero --> ~X). | ||||||||
| 3096 | Value *Or = Builder.CreateOr(X, Y); | ||||||||
| 3097 | return BinaryOperator::CreateXor(Or, ConstantInt::get(Ty, *CV)); | ||||||||
| 3098 | } | ||||||||
| 3099 | |||||||||
| 3100 | // If the operands have no common bits set: | ||||||||
| 3101 | // or (mul X, Y), X --> add (mul X, Y), X --> mul X, (Y + 1) | ||||||||
| 3102 | if (match(&I, | ||||||||
| 3103 | m_c_Or(m_OneUse(m_Mul(m_Value(X), m_Value(Y))), m_Deferred(X))) && | ||||||||
| 3104 | haveNoCommonBitsSet(Op0, Op1, DL)) { | ||||||||
| 3105 | Value *IncrementY = Builder.CreateAdd(Y, ConstantInt::get(Ty, 1)); | ||||||||
| 3106 | return BinaryOperator::CreateMul(X, IncrementY); | ||||||||
| 3107 | } | ||||||||
| 3108 | |||||||||
| 3109 | // X | (X ^ Y) --> X | Y (4 commuted patterns) | ||||||||
| 3110 | if (match(&I, m_c_Or(m_Value(X), m_c_Xor(m_Deferred(X), m_Value(Y))))) | ||||||||
| 3111 | return BinaryOperator::CreateOr(X, Y); | ||||||||
| 3112 | |||||||||
| 3113 | // (A & C) | (B & D) | ||||||||
| 3114 | Value *A, *B, *C, *D; | ||||||||
| 3115 | if (match(Op0, m_And(m_Value(A), m_Value(C))) && | ||||||||
| 3116 | match(Op1, m_And(m_Value(B), m_Value(D)))) { | ||||||||
| 3117 | |||||||||
| 3118 | // (A & C0) | (B & C1) | ||||||||
| 3119 | const APInt *C0, *C1; | ||||||||
| 3120 | if (match(C, m_APInt(C0)) && match(D, m_APInt(C1))) { | ||||||||
| 3121 | Value *X; | ||||||||
| 3122 | if (*C0 == ~*C1) { | ||||||||
| 3123 | // ((X | B) & MaskC) | (B & ~MaskC) -> (X & MaskC) | B | ||||||||
| 3124 | if (match(A, m_c_Or(m_Value(X), m_Specific(B)))) | ||||||||
| 3125 | return BinaryOperator::CreateOr(Builder.CreateAnd(X, *C0), B); | ||||||||
| 3126 | // (A & MaskC) | ((X | A) & ~MaskC) -> (X & ~MaskC) | A | ||||||||
| 3127 | if (match(B, m_c_Or(m_Specific(A), m_Value(X)))) | ||||||||
| 3128 | return BinaryOperator::CreateOr(Builder.CreateAnd(X, *C1), A); | ||||||||
| 3129 | |||||||||
| 3130 | // ((X ^ B) & MaskC) | (B & ~MaskC) -> (X & MaskC) ^ B | ||||||||
| 3131 | if (match(A, m_c_Xor(m_Value(X), m_Specific(B)))) | ||||||||
| 3132 | return BinaryOperator::CreateXor(Builder.CreateAnd(X, *C0), B); | ||||||||
| 3133 | // (A & MaskC) | ((X ^ A) & ~MaskC) -> (X & ~MaskC) ^ A | ||||||||
| 3134 | if (match(B, m_c_Xor(m_Specific(A), m_Value(X)))) | ||||||||
| 3135 | return BinaryOperator::CreateXor(Builder.CreateAnd(X, *C1), A); | ||||||||
| 3136 | } | ||||||||
| 3137 | |||||||||
| 3138 | if ((*C0 & *C1).isZero()) { | ||||||||
| 3139 | // ((X | B) & C0) | (B & C1) --> (X | B) & (C0 | C1) | ||||||||
| 3140 | // iff (C0 & C1) == 0 and (X & ~C0) == 0 | ||||||||
| 3141 | if (match(A, m_c_Or(m_Value(X), m_Specific(B))) && | ||||||||
| 3142 | MaskedValueIsZero(X, ~*C0, 0, &I)) { | ||||||||
| 3143 | Constant *C01 = ConstantInt::get(Ty, *C0 | *C1); | ||||||||
| 3144 | return BinaryOperator::CreateAnd(A, C01); | ||||||||
| 3145 | } | ||||||||
| 3146 | // (A & C0) | ((X | A) & C1) --> (X | A) & (C0 | C1) | ||||||||
| 3147 | // iff (C0 & C1) == 0 and (X & ~C1) == 0 | ||||||||
| 3148 | if (match(B, m_c_Or(m_Value(X), m_Specific(A))) && | ||||||||
| 3149 | MaskedValueIsZero(X, ~*C1, 0, &I)) { | ||||||||
| 3150 | Constant *C01 = ConstantInt::get(Ty, *C0 | *C1); | ||||||||
| 3151 | return BinaryOperator::CreateAnd(B, C01); | ||||||||
| 3152 | } | ||||||||
| 3153 | // ((X | C2) & C0) | ((X | C3) & C1) --> (X | C2 | C3) & (C0 | C1) | ||||||||
| 3154 | // iff (C0 & C1) == 0 and (C2 & ~C0) == 0 and (C3 & ~C1) == 0. | ||||||||
| 3155 | const APInt *C2, *C3; | ||||||||
| 3156 | if (match(A, m_Or(m_Value(X), m_APInt(C2))) && | ||||||||
| 3157 | match(B, m_Or(m_Specific(X), m_APInt(C3))) && | ||||||||
| 3158 | (*C2 & ~*C0).isZero() && (*C3 & ~*C1).isZero()) { | ||||||||
| 3159 | Value *Or = Builder.CreateOr(X, *C2 | *C3, "bitfield"); | ||||||||
| 3160 | Constant *C01 = ConstantInt::get(Ty, *C0 | *C1); | ||||||||
| 3161 | return BinaryOperator::CreateAnd(Or, C01); | ||||||||
| 3162 | } | ||||||||
| 3163 | } | ||||||||
| 3164 | } | ||||||||
| 3165 | |||||||||
| 3166 | // Don't try to form a select if it's unlikely that we'll get rid of at | ||||||||
| 3167 | // least one of the operands. A select is generally more expensive than the | ||||||||
| 3168 | // 'or' that it is replacing. | ||||||||
| 3169 | if (Op0->hasOneUse() || Op1->hasOneUse()) { | ||||||||
| 3170 | // (Cond & C) | (~Cond & D) -> Cond ? C : D, and commuted variants. | ||||||||
| 3171 | if (Value *V = matchSelectFromAndOr(A, C, B, D)) | ||||||||
| 3172 | return replaceInstUsesWith(I, V); | ||||||||
| 3173 | if (Value *V = matchSelectFromAndOr(A, C, D, B)) | ||||||||
| 3174 | return replaceInstUsesWith(I, V); | ||||||||
| 3175 | if (Value *V = matchSelectFromAndOr(C, A, B, D)) | ||||||||
| 3176 | return replaceInstUsesWith(I, V); | ||||||||
| 3177 | if (Value *V = matchSelectFromAndOr(C, A, D, B)) | ||||||||
| 3178 | return replaceInstUsesWith(I, V); | ||||||||
| 3179 | if (Value *V = matchSelectFromAndOr(B, D, A, C)) | ||||||||
| 3180 | return replaceInstUsesWith(I, V); | ||||||||
| 3181 | if (Value *V = matchSelectFromAndOr(B, D, C, A)) | ||||||||
| 3182 | return replaceInstUsesWith(I, V); | ||||||||
| 3183 | if (Value *V = matchSelectFromAndOr(D, B, A, C)) | ||||||||
| 3184 | return replaceInstUsesWith(I, V); | ||||||||
| 3185 | if (Value *V = matchSelectFromAndOr(D, B, C, A)) | ||||||||
| 3186 | return replaceInstUsesWith(I, V); | ||||||||
| 3187 | } | ||||||||
| 3188 | } | ||||||||
| 3189 | |||||||||
| 3190 | if (match(Op0, m_And(m_Value(A), m_Value(C))) && | ||||||||
| 3191 | match(Op1, m_Not(m_Or(m_Value(B), m_Value(D)))) && | ||||||||
| 3192 | (Op0->hasOneUse() || Op1->hasOneUse())) { | ||||||||
| 3193 | // (Cond & C) | ~(Cond | D) -> Cond ? C : ~D | ||||||||
| 3194 | if (Value *V = matchSelectFromAndOr(A, C, B, D, true)) | ||||||||
| 3195 | return replaceInstUsesWith(I, V); | ||||||||
| 3196 | if (Value *V = matchSelectFromAndOr(A, C, D, B, true)) | ||||||||
| 3197 | return replaceInstUsesWith(I, V); | ||||||||
| 3198 | if (Value *V = matchSelectFromAndOr(C, A, B, D, true)) | ||||||||
| 3199 | return replaceInstUsesWith(I, V); | ||||||||
| 3200 | if (Value *V = matchSelectFromAndOr(C, A, D, B, true)) | ||||||||
| 3201 | return replaceInstUsesWith(I, V); | ||||||||
| 3202 | } | ||||||||
| 3203 | |||||||||
| 3204 | // (A ^ B) | ((B ^ C) ^ A) -> (A ^ B) | C | ||||||||
| 3205 | if (match(Op0, m_Xor(m_Value(A), m_Value(B)))) | ||||||||
| 3206 | if (match(Op1, m_Xor(m_Xor(m_Specific(B), m_Value(C)), m_Specific(A)))) | ||||||||
| 3207 | return BinaryOperator::CreateOr(Op0, C); | ||||||||
| 3208 | |||||||||
| 3209 | // ((A ^ C) ^ B) | (B ^ A) -> (B ^ A) | C | ||||||||
| 3210 | if (match(Op0, m_Xor(m_Xor(m_Value(A), m_Value(C)), m_Value(B)))) | ||||||||
| 3211 | if (match(Op1, m_Xor(m_Specific(B), m_Specific(A)))) | ||||||||
| 3212 | return BinaryOperator::CreateOr(Op1, C); | ||||||||
| 3213 | |||||||||
| 3214 | // ((A & B) ^ C) | B -> C | B | ||||||||
| 3215 | if (match(Op0, m_c_Xor(m_c_And(m_Value(A), m_Specific(Op1)), m_Value(C)))) | ||||||||
| 3216 | return BinaryOperator::CreateOr(C, Op1); | ||||||||
| 3217 | |||||||||
| 3218 | // B | ((A & B) ^ C) -> B | C | ||||||||
| 3219 | if (match(Op1, m_c_Xor(m_c_And(m_Value(A), m_Specific(Op0)), m_Value(C)))) | ||||||||
| 3220 | return BinaryOperator::CreateOr(Op0, C); | ||||||||
| 3221 | |||||||||
| 3222 | // ((B | C) & A) | B -> B | (A & C) | ||||||||
| 3223 | if (match(Op0, m_And(m_Or(m_Specific(Op1), m_Value(C)), m_Value(A)))) | ||||||||
| 3224 | return BinaryOperator::CreateOr(Op1, Builder.CreateAnd(A, C)); | ||||||||
| 3225 | |||||||||
| 3226 | if (Instruction *DeMorgan = matchDeMorgansLaws(I, Builder)) | ||||||||
| 3227 | return DeMorgan; | ||||||||
| 3228 | |||||||||
| 3229 | // Canonicalize xor to the RHS. | ||||||||
| 3230 | bool SwappedForXor = false; | ||||||||
| 3231 | if (match(Op0, m_Xor(m_Value(), m_Value()))) { | ||||||||
| 3232 | std::swap(Op0, Op1); | ||||||||
| 3233 | SwappedForXor = true; | ||||||||
| 3234 | } | ||||||||
| 3235 | |||||||||
| 3236 | if (match(Op1, m_Xor(m_Value(A), m_Value(B)))) { | ||||||||
| 3237 | // (A | ?) | (A ^ B) --> (A | ?) | B | ||||||||
| 3238 | // (B | ?) | (A ^ B) --> (B | ?) | A | ||||||||
| 3239 | if (match(Op0, m_c_Or(m_Specific(A), m_Value()))) | ||||||||
| 3240 | return BinaryOperator::CreateOr(Op0, B); | ||||||||
| 3241 | if (match(Op0, m_c_Or(m_Specific(B), m_Value()))) | ||||||||
| 3242 | return BinaryOperator::CreateOr(Op0, A); | ||||||||
| 3243 | |||||||||
| 3244 | // (A & B) | (A ^ B) --> A | B | ||||||||
| 3245 | // (B & A) | (A ^ B) --> A | B | ||||||||
| 3246 | if (match(Op0, m_And(m_Specific(A), m_Specific(B))) || | ||||||||
| 3247 | match(Op0, m_And(m_Specific(B), m_Specific(A)))) | ||||||||
| 3248 | return BinaryOperator::CreateOr(A, B); | ||||||||
| 3249 | |||||||||
| 3250 | // ~A | (A ^ B) --> ~(A & B) | ||||||||
| 3251 | // ~B | (A ^ B) --> ~(A & B) | ||||||||
| 3252 | // The swap above should always make Op0 the 'not'. | ||||||||
| 3253 | if ((Op0->hasOneUse() || Op1->hasOneUse()) && | ||||||||
| 3254 | (match(Op0, m_Not(m_Specific(A))) || match(Op0, m_Not(m_Specific(B))))) | ||||||||
| 3255 | return BinaryOperator::CreateNot(Builder.CreateAnd(A, B)); | ||||||||
| 3256 | |||||||||
| 3257 | // Same as above, but peek through an 'and' to the common operand: | ||||||||
| 3258 | // ~(A & ?) | (A ^ B) --> ~((A & ?) & B) | ||||||||
| 3259 | // ~(B & ?) | (A ^ B) --> ~((B & ?) & A) | ||||||||
| 3260 | Instruction *And; | ||||||||
| 3261 | if ((Op0->hasOneUse() || Op1->hasOneUse()) && | ||||||||
| 3262 | match(Op0, m_Not(m_CombineAnd(m_Instruction(And), | ||||||||
| 3263 | m_c_And(m_Specific(A), m_Value()))))) | ||||||||
| 3264 | return BinaryOperator::CreateNot(Builder.CreateAnd(And, B)); | ||||||||
| 3265 | if ((Op0->hasOneUse() || Op1->hasOneUse()) && | ||||||||
| 3266 | match(Op0, m_Not(m_CombineAnd(m_Instruction(And), | ||||||||
| 3267 | m_c_And(m_Specific(B), m_Value()))))) | ||||||||
| 3268 | return BinaryOperator::CreateNot(Builder.CreateAnd(And, A)); | ||||||||
| 3269 | |||||||||
| 3270 | // (~A | C) | (A ^ B) --> ~(A & B) | C | ||||||||
| 3271 | // (~B | C) | (A ^ B) --> ~(A & B) | C | ||||||||
| 3272 | if (Op0->hasOneUse() && Op1->hasOneUse() && | ||||||||
| 3273 | (match(Op0, m_c_Or(m_Not(m_Specific(A)), m_Value(C))) || | ||||||||
| 3274 | match(Op0, m_c_Or(m_Not(m_Specific(B)), m_Value(C))))) { | ||||||||
| 3275 | Value *Nand = Builder.CreateNot(Builder.CreateAnd(A, B), "nand"); | ||||||||
| 3276 | return BinaryOperator::CreateOr(Nand, C); | ||||||||
| 3277 | } | ||||||||
| 3278 | |||||||||
| 3279 | // A | (~A ^ B) --> ~B | A | ||||||||
| 3280 | // B | (A ^ ~B) --> ~A | B | ||||||||
| 3281 | if (Op1->hasOneUse() && match(A, m_Not(m_Specific(Op0)))) { | ||||||||
| 3282 | Value *NotB = Builder.CreateNot(B, B->getName() + ".not"); | ||||||||
| 3283 | return BinaryOperator::CreateOr(NotB, Op0); | ||||||||
| 3284 | } | ||||||||
| 3285 | if (Op1->hasOneUse() && match(B, m_Not(m_Specific(Op0)))) { | ||||||||
| 3286 | Value *NotA = Builder.CreateNot(A, A->getName() + ".not"); | ||||||||
| 3287 | return BinaryOperator::CreateOr(NotA, Op0); | ||||||||
| 3288 | } | ||||||||
| 3289 | } | ||||||||
| 3290 | |||||||||
| 3291 | // A | ~(A | B) -> A | ~B | ||||||||
| 3292 | // A | ~(A ^ B) -> A | ~B | ||||||||
| 3293 | if (match(Op1, m_Not(m_Value(A)))) | ||||||||
| 3294 | if (BinaryOperator *B = dyn_cast<BinaryOperator>(A)) | ||||||||
| 3295 | if ((Op0 == B->getOperand(0) || Op0 == B->getOperand(1)) && | ||||||||
| 3296 | Op1->hasOneUse() && (B->getOpcode() == Instruction::Or || | ||||||||
| 3297 | B->getOpcode() == Instruction::Xor)) { | ||||||||
| 3298 | Value *NotOp = Op0 == B->getOperand(0) ? B->getOperand(1) : | ||||||||
| 3299 | B->getOperand(0); | ||||||||
| 3300 | Value *Not = Builder.CreateNot(NotOp, NotOp->getName() + ".not"); | ||||||||
| 3301 | return BinaryOperator::CreateOr(Not, Op0); | ||||||||
| 3302 | } | ||||||||
| 3303 | |||||||||
| 3304 | if (SwappedForXor) | ||||||||
| 3305 | std::swap(Op0, Op1); | ||||||||
| 3306 | |||||||||
| 3307 | { | ||||||||
| 3308 | ICmpInst *LHS = dyn_cast<ICmpInst>(Op0); | ||||||||
| 3309 | ICmpInst *RHS = dyn_cast<ICmpInst>(Op1); | ||||||||
| 3310 | if (LHS && RHS) | ||||||||
| 3311 | if (Value *Res = foldAndOrOfICmps(LHS, RHS, I, /* IsAnd */ false)) | ||||||||
| 3312 | return replaceInstUsesWith(I, Res); | ||||||||
| 3313 | |||||||||
| 3314 | // TODO: Make this recursive; it's a little tricky because an arbitrary | ||||||||
| 3315 | // number of 'or' instructions might have to be created. | ||||||||
| 3316 | Value *X, *Y; | ||||||||
| 3317 | if (LHS && match(Op1, m_OneUse(m_LogicalOr(m_Value(X), m_Value(Y))))) { | ||||||||
| 3318 | bool IsLogical = isa<SelectInst>(Op1); | ||||||||
| 3319 | // LHS | (X || Y) --> (LHS || X) || Y | ||||||||
| 3320 | if (auto *Cmp = dyn_cast<ICmpInst>(X)) | ||||||||
| 3321 | if (Value *Res = | ||||||||
| 3322 | foldAndOrOfICmps(LHS, Cmp, I, /* IsAnd */ false, IsLogical)) | ||||||||
| 3323 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 3324 | ? Builder.CreateLogicalOr(Res, Y) | ||||||||
| 3325 | : Builder.CreateOr(Res, Y)); | ||||||||
| 3326 | // LHS | (X || Y) --> X || (LHS | Y) | ||||||||
| 3327 | if (auto *Cmp = dyn_cast<ICmpInst>(Y)) | ||||||||
| 3328 | if (Value *Res = foldAndOrOfICmps(LHS, Cmp, I, /* IsAnd */ false, | ||||||||
| 3329 | /* IsLogical */ false)) | ||||||||
| 3330 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 3331 | ? Builder.CreateLogicalOr(X, Res) | ||||||||
| 3332 | : Builder.CreateOr(X, Res)); | ||||||||
| 3333 | } | ||||||||
| 3334 | if (RHS && match(Op0, m_OneUse(m_LogicalOr(m_Value(X), m_Value(Y))))) { | ||||||||
| 3335 | bool IsLogical = isa<SelectInst>(Op0); | ||||||||
| 3336 | // (X || Y) | RHS --> (X || RHS) || Y | ||||||||
| 3337 | if (auto *Cmp = dyn_cast<ICmpInst>(X)) | ||||||||
| 3338 | if (Value *Res = | ||||||||
| 3339 | foldAndOrOfICmps(Cmp, RHS, I, /* IsAnd */ false, IsLogical)) | ||||||||
| 3340 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 3341 | ? Builder.CreateLogicalOr(Res, Y) | ||||||||
| 3342 | : Builder.CreateOr(Res, Y)); | ||||||||
| 3343 | // (X || Y) | RHS --> X || (Y | RHS) | ||||||||
| 3344 | if (auto *Cmp = dyn_cast<ICmpInst>(Y)) | ||||||||
| 3345 | if (Value *Res = foldAndOrOfICmps(Cmp, RHS, I, /* IsAnd */ false, | ||||||||
| 3346 | /* IsLogical */ false)) | ||||||||
| 3347 | return replaceInstUsesWith(I, IsLogical | ||||||||
| 3348 | ? Builder.CreateLogicalOr(X, Res) | ||||||||
| 3349 | : Builder.CreateOr(X, Res)); | ||||||||
| 3350 | } | ||||||||
| 3351 | } | ||||||||
| 3352 | |||||||||
| 3353 | if (FCmpInst *LHS = dyn_cast<FCmpInst>(I.getOperand(0))) | ||||||||
| 3354 | if (FCmpInst *RHS = dyn_cast<FCmpInst>(I.getOperand(1))) | ||||||||
| 3355 | if (Value *Res = foldLogicOfFCmps(LHS, RHS, /*IsAnd*/ false)) | ||||||||
| 3356 | return replaceInstUsesWith(I, Res); | ||||||||
| 3357 | |||||||||
| 3358 | if (Instruction *FoldedFCmps = reassociateFCmps(I, Builder)) | ||||||||
| 3359 | return FoldedFCmps; | ||||||||
| 3360 | |||||||||
| 3361 | if (Instruction *CastedOr = foldCastedBitwiseLogic(I)) | ||||||||
| 3362 | return CastedOr; | ||||||||
| 3363 | |||||||||
| 3364 | if (Instruction *Sel = foldBinopOfSextBoolToSelect(I)) | ||||||||
| 3365 | return Sel; | ||||||||
| 3366 | |||||||||
| 3367 | // or(sext(A), B) / or(B, sext(A)) --> A ? -1 : B, where A is i1 or <N x i1>. | ||||||||
| 3368 | // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold | ||||||||
| 3369 | // with binop identity constant. But creating a select with non-constant | ||||||||
| 3370 | // arm may not be reversible due to poison semantics. Is that a good | ||||||||
| 3371 | // canonicalization? | ||||||||
| 3372 | if (match(Op0, m_OneUse(m_SExt(m_Value(A)))) && | ||||||||
| 3373 | A->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 3374 | return SelectInst::Create(A, ConstantInt::getAllOnesValue(Ty), Op1); | ||||||||
| 3375 | if (match(Op1, m_OneUse(m_SExt(m_Value(A)))) && | ||||||||
| 3376 | A->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 3377 | return SelectInst::Create(A, ConstantInt::getAllOnesValue(Ty), Op0); | ||||||||
| 3378 | |||||||||
| 3379 | // Note: If we've gotten to the point of visiting the outer OR, then the | ||||||||
| 3380 | // inner one couldn't be simplified. If it was a constant, then it won't | ||||||||
| 3381 | // be simplified by a later pass either, so we try swapping the inner/outer | ||||||||
| 3382 | // ORs in the hopes that we'll be able to simplify it this way. | ||||||||
| 3383 | // (X|C) | V --> (X|V) | C | ||||||||
| 3384 | ConstantInt *CI; | ||||||||
| 3385 | if (Op0->hasOneUse() && !match(Op1, m_ConstantInt()) && | ||||||||
| 3386 | match(Op0, m_Or(m_Value(A), m_ConstantInt(CI)))) { | ||||||||
| 3387 | Value *Inner = Builder.CreateOr(A, Op1); | ||||||||
| 3388 | Inner->takeName(Op0); | ||||||||
| 3389 | return BinaryOperator::CreateOr(Inner, CI); | ||||||||
| 3390 | } | ||||||||
| 3391 | |||||||||
| 3392 | // Change (or (bool?A:B),(bool?C:D)) --> (bool?(or A,C):(or B,D)) | ||||||||
| 3393 | // Since this OR statement hasn't been optimized further yet, we hope | ||||||||
| 3394 | // that this transformation will allow the new ORs to be optimized. | ||||||||
| 3395 | { | ||||||||
| 3396 | Value *X = nullptr, *Y = nullptr; | ||||||||
| 3397 | if (Op0->hasOneUse() && Op1->hasOneUse() && | ||||||||
| 3398 | match(Op0, m_Select(m_Value(X), m_Value(A), m_Value(B))) && | ||||||||
| 3399 | match(Op1, m_Select(m_Value(Y), m_Value(C), m_Value(D))) && X == Y) { | ||||||||
| 3400 | Value *orTrue = Builder.CreateOr(A, C); | ||||||||
| 3401 | Value *orFalse = Builder.CreateOr(B, D); | ||||||||
| 3402 | return SelectInst::Create(X, orTrue, orFalse); | ||||||||
| 3403 | } | ||||||||
| 3404 | } | ||||||||
| 3405 | |||||||||
| 3406 | // or(ashr(subNSW(Y, X), ScalarSizeInBits(Y) - 1), X) --> X s> Y ? -1 : X. | ||||||||
| 3407 | { | ||||||||
| 3408 | Value *X, *Y; | ||||||||
| 3409 | if (match(&I, m_c_Or(m_OneUse(m_AShr( | ||||||||
| 3410 | m_NSWSub(m_Value(Y), m_Value(X)), | ||||||||
| 3411 | m_SpecificInt(Ty->getScalarSizeInBits() - 1))), | ||||||||
| 3412 | m_Deferred(X)))) { | ||||||||
| 3413 | Value *NewICmpInst = Builder.CreateICmpSGT(X, Y); | ||||||||
| 3414 | Value *AllOnes = ConstantInt::getAllOnesValue(Ty); | ||||||||
| 3415 | return SelectInst::Create(NewICmpInst, AllOnes, X); | ||||||||
| 3416 | } | ||||||||
| 3417 | } | ||||||||
| 3418 | |||||||||
| 3419 | if (Instruction *V = | ||||||||
| 3420 | canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I)) | ||||||||
| 3421 | return V; | ||||||||
| 3422 | |||||||||
| 3423 | CmpInst::Predicate Pred; | ||||||||
| 3424 | Value *Mul, *Ov, *MulIsNotZero, *UMulWithOv; | ||||||||
| 3425 | // Check if the OR weakens the overflow condition for umul.with.overflow by | ||||||||
| 3426 | // treating any non-zero result as overflow. In that case, we overflow if both | ||||||||
| 3427 | // umul.with.overflow operands are != 0, as in that case the result can only | ||||||||
| 3428 | // be 0, iff the multiplication overflows. | ||||||||
| 3429 | if (match(&I, | ||||||||
| 3430 | m_c_Or(m_CombineAnd(m_ExtractValue<1>(m_Value(UMulWithOv)), | ||||||||
| 3431 | m_Value(Ov)), | ||||||||
| 3432 | m_CombineAnd(m_ICmp(Pred, | ||||||||
| 3433 | m_CombineAnd(m_ExtractValue<0>( | ||||||||
| 3434 | m_Deferred(UMulWithOv)), | ||||||||
| 3435 | m_Value(Mul)), | ||||||||
| 3436 | m_ZeroInt()), | ||||||||
| 3437 | m_Value(MulIsNotZero)))) && | ||||||||
| 3438 | (Ov->hasOneUse() || (MulIsNotZero->hasOneUse() && Mul->hasOneUse())) && | ||||||||
| 3439 | Pred == CmpInst::ICMP_NE) { | ||||||||
| 3440 | Value *A, *B; | ||||||||
| 3441 | if (match(UMulWithOv, m_Intrinsic<Intrinsic::umul_with_overflow>( | ||||||||
| 3442 | m_Value(A), m_Value(B)))) { | ||||||||
| 3443 | Value *NotNullA = Builder.CreateIsNotNull(A); | ||||||||
| 3444 | Value *NotNullB = Builder.CreateIsNotNull(B); | ||||||||
| 3445 | return BinaryOperator::CreateAnd(NotNullA, NotNullB); | ||||||||
| 3446 | } | ||||||||
| 3447 | } | ||||||||
| 3448 | |||||||||
| 3449 | // (~x) | y --> ~(x & (~y)) iff that gets rid of inversions | ||||||||
| 3450 | if (sinkNotIntoOtherHandOfLogicalOp(I)) | ||||||||
| 3451 | return &I; | ||||||||
| 3452 | |||||||||
| 3453 | // Improve "get low bit mask up to and including bit X" pattern: | ||||||||
| 3454 | // (1 << X) | ((1 << X) + -1) --> -1 l>> (bitwidth(x) - 1 - X) | ||||||||
| 3455 | if (match(&I, m_c_Or(m_Add(m_Shl(m_One(), m_Value(X)), m_AllOnes()), | ||||||||
| 3456 | m_Shl(m_One(), m_Deferred(X)))) && | ||||||||
| 3457 | match(&I, m_c_Or(m_OneUse(m_Value()), m_Value()))) { | ||||||||
| 3458 | Value *Sub = Builder.CreateSub( | ||||||||
| 3459 | ConstantInt::get(Ty, Ty->getScalarSizeInBits() - 1), X); | ||||||||
| 3460 | return BinaryOperator::CreateLShr(Constant::getAllOnesValue(Ty), Sub); | ||||||||
| 3461 | } | ||||||||
| 3462 | |||||||||
| 3463 | // An or recurrence w/loop invariant step is equivelent to (or start, step) | ||||||||
| 3464 | PHINode *PN = nullptr; | ||||||||
| 3465 | Value *Start = nullptr, *Step = nullptr; | ||||||||
| 3466 | if (matchSimpleRecurrence(&I, PN, Start, Step) && DT.dominates(Step, PN)) | ||||||||
| 3467 | return replaceInstUsesWith(I, Builder.CreateOr(Start, Step)); | ||||||||
| 3468 | |||||||||
| 3469 | // (A & B) | (C | D) or (C | D) | (A & B) | ||||||||
| 3470 | // Can be combined if C or D is of type (A/B & X) | ||||||||
| 3471 | if (match(&I, m_c_Or(m_OneUse(m_And(m_Value(A), m_Value(B))), | ||||||||
| 3472 | m_OneUse(m_Or(m_Value(C), m_Value(D)))))) { | ||||||||
| 3473 | // (A & B) | (C | ?) -> C | (? | (A & B)) | ||||||||
| 3474 | // (A & B) | (C | ?) -> C | (? | (A & B)) | ||||||||
| 3475 | // (A & B) | (C | ?) -> C | (? | (A & B)) | ||||||||
| 3476 | // (A & B) | (C | ?) -> C | (? | (A & B)) | ||||||||
| 3477 | // (C | ?) | (A & B) -> C | (? | (A & B)) | ||||||||
| 3478 | // (C | ?) | (A & B) -> C | (? | (A & B)) | ||||||||
| 3479 | // (C | ?) | (A & B) -> C | (? | (A & B)) | ||||||||
| 3480 | // (C | ?) | (A & B) -> C | (? | (A & B)) | ||||||||
| 3481 | if (match(D, m_OneUse(m_c_And(m_Specific(A), m_Value()))) || | ||||||||
| 3482 | match(D, m_OneUse(m_c_And(m_Specific(B), m_Value())))) | ||||||||
| 3483 | return BinaryOperator::CreateOr( | ||||||||
| 3484 | C, Builder.CreateOr(D, Builder.CreateAnd(A, B))); | ||||||||
| 3485 | // (A & B) | (? | D) -> (? | (A & B)) | D | ||||||||
| 3486 | // (A & B) | (? | D) -> (? | (A & B)) | D | ||||||||
| 3487 | // (A & B) | (? | D) -> (? | (A & B)) | D | ||||||||
| 3488 | // (A & B) | (? | D) -> (? | (A & B)) | D | ||||||||
| 3489 | // (? | D) | (A & B) -> (? | (A & B)) | D | ||||||||
| 3490 | // (? | D) | (A & B) -> (? | (A & B)) | D | ||||||||
| 3491 | // (? | D) | (A & B) -> (? | (A & B)) | D | ||||||||
| 3492 | // (? | D) | (A & B) -> (? | (A & B)) | D | ||||||||
| 3493 | if (match(C, m_OneUse(m_c_And(m_Specific(A), m_Value()))) || | ||||||||
| 3494 | match(C, m_OneUse(m_c_And(m_Specific(B), m_Value())))) | ||||||||
| 3495 | return BinaryOperator::CreateOr( | ||||||||
| 3496 | Builder.CreateOr(C, Builder.CreateAnd(A, B)), D); | ||||||||
| 3497 | } | ||||||||
| 3498 | |||||||||
| 3499 | if (Instruction *R = reassociateForUses(I, Builder)) | ||||||||
| 3500 | return R; | ||||||||
| 3501 | |||||||||
| 3502 | if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder)) | ||||||||
| 3503 | return Canonicalized; | ||||||||
| 3504 | |||||||||
| 3505 | if (Instruction *Folded = foldLogicOfIsFPClass(I, Op0, Op1)) | ||||||||
| 3506 | return Folded; | ||||||||
| 3507 | |||||||||
| 3508 | return nullptr; | ||||||||
| 3509 | } | ||||||||
| 3510 | |||||||||
| 3511 | /// A ^ B can be specified using other logic ops in a variety of patterns. We | ||||||||
| 3512 | /// can fold these early and efficiently by morphing an existing instruction. | ||||||||
| 3513 | static Instruction *foldXorToXor(BinaryOperator &I, | ||||||||
| 3514 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3515 | assert(I.getOpcode() == Instruction::Xor)(static_cast <bool> (I.getOpcode() == Instruction::Xor) ? void (0) : __assert_fail ("I.getOpcode() == Instruction::Xor" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 3515 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3516 | Value *Op0 = I.getOperand(0); | ||||||||
| 3517 | Value *Op1 = I.getOperand(1); | ||||||||
| 3518 | Value *A, *B; | ||||||||
| 3519 | |||||||||
| 3520 | // There are 4 commuted variants for each of the basic patterns. | ||||||||
| 3521 | |||||||||
| 3522 | // (A & B) ^ (A | B) -> A ^ B | ||||||||
| 3523 | // (A & B) ^ (B | A) -> A ^ B | ||||||||
| 3524 | // (A | B) ^ (A & B) -> A ^ B | ||||||||
| 3525 | // (A | B) ^ (B & A) -> A ^ B | ||||||||
| 3526 | if (match(&I, m_c_Xor(m_And(m_Value(A), m_Value(B)), | ||||||||
| 3527 | m_c_Or(m_Deferred(A), m_Deferred(B))))) | ||||||||
| 3528 | return BinaryOperator::CreateXor(A, B); | ||||||||
| 3529 | |||||||||
| 3530 | // (A | ~B) ^ (~A | B) -> A ^ B | ||||||||
| 3531 | // (~B | A) ^ (~A | B) -> A ^ B | ||||||||
| 3532 | // (~A | B) ^ (A | ~B) -> A ^ B | ||||||||
| 3533 | // (B | ~A) ^ (A | ~B) -> A ^ B | ||||||||
| 3534 | if (match(&I, m_Xor(m_c_Or(m_Value(A), m_Not(m_Value(B))), | ||||||||
| 3535 | m_c_Or(m_Not(m_Deferred(A)), m_Deferred(B))))) | ||||||||
| 3536 | return BinaryOperator::CreateXor(A, B); | ||||||||
| 3537 | |||||||||
| 3538 | // (A & ~B) ^ (~A & B) -> A ^ B | ||||||||
| 3539 | // (~B & A) ^ (~A & B) -> A ^ B | ||||||||
| 3540 | // (~A & B) ^ (A & ~B) -> A ^ B | ||||||||
| 3541 | // (B & ~A) ^ (A & ~B) -> A ^ B | ||||||||
| 3542 | if (match(&I, m_Xor(m_c_And(m_Value(A), m_Not(m_Value(B))), | ||||||||
| 3543 | m_c_And(m_Not(m_Deferred(A)), m_Deferred(B))))) | ||||||||
| 3544 | return BinaryOperator::CreateXor(A, B); | ||||||||
| 3545 | |||||||||
| 3546 | // For the remaining cases we need to get rid of one of the operands. | ||||||||
| 3547 | if (!Op0->hasOneUse() && !Op1->hasOneUse()) | ||||||||
| 3548 | return nullptr; | ||||||||
| 3549 | |||||||||
| 3550 | // (A | B) ^ ~(A & B) -> ~(A ^ B) | ||||||||
| 3551 | // (A | B) ^ ~(B & A) -> ~(A ^ B) | ||||||||
| 3552 | // (A & B) ^ ~(A | B) -> ~(A ^ B) | ||||||||
| 3553 | // (A & B) ^ ~(B | A) -> ~(A ^ B) | ||||||||
| 3554 | // Complexity sorting ensures the not will be on the right side. | ||||||||
| 3555 | if ((match(Op0, m_Or(m_Value(A), m_Value(B))) && | ||||||||
| 3556 | match(Op1, m_Not(m_c_And(m_Specific(A), m_Specific(B))))) || | ||||||||
| 3557 | (match(Op0, m_And(m_Value(A), m_Value(B))) && | ||||||||
| 3558 | match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))) | ||||||||
| 3559 | return BinaryOperator::CreateNot(Builder.CreateXor(A, B)); | ||||||||
| 3560 | |||||||||
| 3561 | return nullptr; | ||||||||
| 3562 | } | ||||||||
| 3563 | |||||||||
| 3564 | Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS, | ||||||||
| 3565 | BinaryOperator &I) { | ||||||||
| 3566 | assert(I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS &&(static_cast <bool> (I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS && I.getOperand(1) == RHS && "Should be 'xor' with these operands") ? void (0) : __assert_fail ("I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS && I.getOperand(1) == RHS && \"Should be 'xor' with these operands\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 3567 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 3567 | I.getOperand(1) == RHS && "Should be 'xor' with these operands")(static_cast <bool> (I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS && I.getOperand(1) == RHS && "Should be 'xor' with these operands") ? void (0) : __assert_fail ("I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS && I.getOperand(1) == RHS && \"Should be 'xor' with these operands\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 3567 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3568 | |||||||||
| 3569 | ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate(); | ||||||||
| 3570 | Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1); | ||||||||
| 3571 | Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1); | ||||||||
| 3572 | |||||||||
| 3573 | if (predicatesFoldable(PredL, PredR)) { | ||||||||
| 3574 | if (LHS0 == RHS1 && LHS1 == RHS0) { | ||||||||
| 3575 | std::swap(LHS0, LHS1); | ||||||||
| 3576 | PredL = ICmpInst::getSwappedPredicate(PredL); | ||||||||
| 3577 | } | ||||||||
| 3578 | if (LHS0 == RHS0 && LHS1 == RHS1) { | ||||||||
| 3579 | // (icmp1 A, B) ^ (icmp2 A, B) --> (icmp3 A, B) | ||||||||
| 3580 | unsigned Code = getICmpCode(PredL) ^ getICmpCode(PredR); | ||||||||
| 3581 | bool IsSigned = LHS->isSigned() || RHS->isSigned(); | ||||||||
| 3582 | return getNewICmpValue(Code, IsSigned, LHS0, LHS1, Builder); | ||||||||
| 3583 | } | ||||||||
| 3584 | } | ||||||||
| 3585 | |||||||||
| 3586 | // TODO: This can be generalized to compares of non-signbits using | ||||||||
| 3587 | // decomposeBitTestICmp(). It could be enhanced more by using (something like) | ||||||||
| 3588 | // foldLogOpOfMaskedICmps(). | ||||||||
| 3589 | const APInt *LC, *RC; | ||||||||
| 3590 | if (match(LHS1, m_APInt(LC)) && match(RHS1, m_APInt(RC)) && | ||||||||
| 3591 | LHS0->getType() == RHS0->getType() && | ||||||||
| 3592 | LHS0->getType()->isIntOrIntVectorTy() && | ||||||||
| 3593 | (LHS->hasOneUse() || RHS->hasOneUse())) { | ||||||||
| 3594 | // Convert xor of signbit tests to signbit test of xor'd values: | ||||||||
| 3595 | // (X > -1) ^ (Y > -1) --> (X ^ Y) < 0 | ||||||||
| 3596 | // (X < 0) ^ (Y < 0) --> (X ^ Y) < 0 | ||||||||
| 3597 | // (X > -1) ^ (Y < 0) --> (X ^ Y) > -1 | ||||||||
| 3598 | // (X < 0) ^ (Y > -1) --> (X ^ Y) > -1 | ||||||||
| 3599 | bool TrueIfSignedL, TrueIfSignedR; | ||||||||
| 3600 | if (isSignBitCheck(PredL, *LC, TrueIfSignedL) && | ||||||||
| 3601 | isSignBitCheck(PredR, *RC, TrueIfSignedR)) { | ||||||||
| 3602 | Value *XorLR = Builder.CreateXor(LHS0, RHS0); | ||||||||
| 3603 | return TrueIfSignedL == TrueIfSignedR ? Builder.CreateIsNeg(XorLR) : | ||||||||
| 3604 | Builder.CreateIsNotNeg(XorLR); | ||||||||
| 3605 | } | ||||||||
| 3606 | |||||||||
| 3607 | // (X > C) ^ (X < C + 2) --> X != C + 1 | ||||||||
| 3608 | // (X < C + 2) ^ (X > C) --> X != C + 1 | ||||||||
| 3609 | // Considering the correctness of this pattern, we should avoid that C is | ||||||||
| 3610 | // non-negative and C + 2 is negative, although it will be matched by other | ||||||||
| 3611 | // patterns. | ||||||||
| 3612 | const APInt *C1, *C2; | ||||||||
| 3613 | if ((PredL == CmpInst::ICMP_SGT && match(LHS1, m_APInt(C1)) && | ||||||||
| 3614 | PredR == CmpInst::ICMP_SLT && match(RHS1, m_APInt(C2))) || | ||||||||
| 3615 | (PredL == CmpInst::ICMP_SLT && match(LHS1, m_APInt(C2)) && | ||||||||
| 3616 | PredR == CmpInst::ICMP_SGT && match(RHS1, m_APInt(C1)))) | ||||||||
| 3617 | if (LHS0 == RHS0 && *C1 + 2 == *C2 && | ||||||||
| 3618 | (C1->isNegative() || C2->isNonNegative())) | ||||||||
| 3619 | return Builder.CreateICmpNE(LHS0, | ||||||||
| 3620 | ConstantInt::get(LHS0->getType(), *C1 + 1)); | ||||||||
| 3621 | } | ||||||||
| 3622 | |||||||||
| 3623 | // Instead of trying to imitate the folds for and/or, decompose this 'xor' | ||||||||
| 3624 | // into those logic ops. That is, try to turn this into an and-of-icmps | ||||||||
| 3625 | // because we have many folds for that pattern. | ||||||||
| 3626 | // | ||||||||
| 3627 | // This is based on a truth table definition of xor: | ||||||||
| 3628 | // X ^ Y --> (X | Y) & !(X & Y) | ||||||||
| 3629 | if (Value *OrICmp = simplifyBinOp(Instruction::Or, LHS, RHS, SQ)) { | ||||||||
| 3630 | // TODO: If OrICmp is true, then the definition of xor simplifies to !(X&Y). | ||||||||
| 3631 | // TODO: If OrICmp is false, the whole thing is false (InstSimplify?). | ||||||||
| 3632 | if (Value *AndICmp = simplifyBinOp(Instruction::And, LHS, RHS, SQ)) { | ||||||||
| 3633 | // TODO: Independently handle cases where the 'and' side is a constant. | ||||||||
| 3634 | ICmpInst *X = nullptr, *Y = nullptr; | ||||||||
| 3635 | if (OrICmp == LHS && AndICmp == RHS) { | ||||||||
| 3636 | // (LHS | RHS) & !(LHS & RHS) --> LHS & !RHS --> X & !Y | ||||||||
| 3637 | X = LHS; | ||||||||
| 3638 | Y = RHS; | ||||||||
| 3639 | } | ||||||||
| 3640 | if (OrICmp == RHS && AndICmp == LHS) { | ||||||||
| 3641 | // !(LHS & RHS) & (LHS | RHS) --> !LHS & RHS --> !Y & X | ||||||||
| 3642 | X = RHS; | ||||||||
| 3643 | Y = LHS; | ||||||||
| 3644 | } | ||||||||
| 3645 | if (X && Y && (Y->hasOneUse() || canFreelyInvertAllUsersOf(Y, &I))) { | ||||||||
| 3646 | // Invert the predicate of 'Y', thus inverting its output. | ||||||||
| 3647 | Y->setPredicate(Y->getInversePredicate()); | ||||||||
| 3648 | // So, are there other uses of Y? | ||||||||
| 3649 | if (!Y->hasOneUse()) { | ||||||||
| 3650 | // We need to adapt other uses of Y though. Get a value that matches | ||||||||
| 3651 | // the original value of Y before inversion. While this increases | ||||||||
| 3652 | // immediate instruction count, we have just ensured that all the | ||||||||
| 3653 | // users are freely-invertible, so that 'not' *will* get folded away. | ||||||||
| 3654 | BuilderTy::InsertPointGuard Guard(Builder); | ||||||||
| 3655 | // Set insertion point to right after the Y. | ||||||||
| 3656 | Builder.SetInsertPoint(Y->getParent(), ++(Y->getIterator())); | ||||||||
| 3657 | Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not"); | ||||||||
| 3658 | // Replace all uses of Y (excluding the one in NotY!) with NotY. | ||||||||
| 3659 | Worklist.pushUsersToWorkList(*Y); | ||||||||
| 3660 | Y->replaceUsesWithIf(NotY, | ||||||||
| 3661 | [NotY](Use &U) { return U.getUser() != NotY; }); | ||||||||
| 3662 | } | ||||||||
| 3663 | // All done. | ||||||||
| 3664 | return Builder.CreateAnd(LHS, RHS); | ||||||||
| 3665 | } | ||||||||
| 3666 | } | ||||||||
| 3667 | } | ||||||||
| 3668 | |||||||||
| 3669 | return nullptr; | ||||||||
| 3670 | } | ||||||||
| 3671 | |||||||||
| 3672 | /// If we have a masked merge, in the canonical form of: | ||||||||
| 3673 | /// (assuming that A only has one use.) | ||||||||
| 3674 | /// | A | |B| | ||||||||
| 3675 | /// ((x ^ y) & M) ^ y | ||||||||
| 3676 | /// | D | | ||||||||
| 3677 | /// * If M is inverted: | ||||||||
| 3678 | /// | D | | ||||||||
| 3679 | /// ((x ^ y) & ~M) ^ y | ||||||||
| 3680 | /// We can canonicalize by swapping the final xor operand | ||||||||
| 3681 | /// to eliminate the 'not' of the mask. | ||||||||
| 3682 | /// ((x ^ y) & M) ^ x | ||||||||
| 3683 | /// * If M is a constant, and D has one use, we transform to 'and' / 'or' ops | ||||||||
| 3684 | /// because that shortens the dependency chain and improves analysis: | ||||||||
| 3685 | /// (x & M) | (y & ~M) | ||||||||
| 3686 | static Instruction *visitMaskedMerge(BinaryOperator &I, | ||||||||
| 3687 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3688 | Value *B, *X, *D; | ||||||||
| 3689 | Value *M; | ||||||||
| 3690 | if (!match(&I, m_c_Xor(m_Value(B), | ||||||||
| 3691 | m_OneUse(m_c_And( | ||||||||
| 3692 | m_CombineAnd(m_c_Xor(m_Deferred(B), m_Value(X)), | ||||||||
| 3693 | m_Value(D)), | ||||||||
| 3694 | m_Value(M)))))) | ||||||||
| 3695 | return nullptr; | ||||||||
| 3696 | |||||||||
| 3697 | Value *NotM; | ||||||||
| 3698 | if (match(M, m_Not(m_Value(NotM)))) { | ||||||||
| 3699 | // De-invert the mask and swap the value in B part. | ||||||||
| 3700 | Value *NewA = Builder.CreateAnd(D, NotM); | ||||||||
| 3701 | return BinaryOperator::CreateXor(NewA, X); | ||||||||
| 3702 | } | ||||||||
| 3703 | |||||||||
| 3704 | Constant *C; | ||||||||
| 3705 | if (D->hasOneUse() && match(M, m_Constant(C))) { | ||||||||
| 3706 | // Propagating undef is unsafe. Clamp undef elements to -1. | ||||||||
| 3707 | Type *EltTy = C->getType()->getScalarType(); | ||||||||
| 3708 | C = Constant::replaceUndefsWith(C, ConstantInt::getAllOnesValue(EltTy)); | ||||||||
| 3709 | // Unfold. | ||||||||
| 3710 | Value *LHS = Builder.CreateAnd(X, C); | ||||||||
| 3711 | Value *NotC = Builder.CreateNot(C); | ||||||||
| 3712 | Value *RHS = Builder.CreateAnd(B, NotC); | ||||||||
| 3713 | return BinaryOperator::CreateOr(LHS, RHS); | ||||||||
| 3714 | } | ||||||||
| 3715 | |||||||||
| 3716 | return nullptr; | ||||||||
| 3717 | } | ||||||||
| 3718 | |||||||||
| 3719 | // Transform | ||||||||
| 3720 | // ~(x ^ y) | ||||||||
| 3721 | // into: | ||||||||
| 3722 | // (~x) ^ y | ||||||||
| 3723 | // or into | ||||||||
| 3724 | // x ^ (~y) | ||||||||
| 3725 | static Instruction *sinkNotIntoXor(BinaryOperator &I, Value *X, Value *Y, | ||||||||
| 3726 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3727 | // We only want to do the transform if it is free to do. | ||||||||
| 3728 | if (InstCombiner::isFreeToInvert(X, X->hasOneUse())) { | ||||||||
| 3729 | // Ok, good. | ||||||||
| 3730 | } else if (InstCombiner::isFreeToInvert(Y, Y->hasOneUse())) { | ||||||||
| 3731 | std::swap(X, Y); | ||||||||
| 3732 | } else | ||||||||
| 3733 | return nullptr; | ||||||||
| 3734 | |||||||||
| 3735 | Value *NotX = Builder.CreateNot(X, X->getName() + ".not"); | ||||||||
| 3736 | return BinaryOperator::CreateXor(NotX, Y, I.getName() + ".demorgan"); | ||||||||
| 3737 | } | ||||||||
| 3738 | |||||||||
| 3739 | static Instruction *foldNotXor(BinaryOperator &I, | ||||||||
| 3740 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3741 | Value *X, *Y; | ||||||||
| 3742 | // FIXME: one-use check is not needed in general, but currently we are unable | ||||||||
| 3743 | // to fold 'not' into 'icmp', if that 'icmp' has multiple uses. (D35182) | ||||||||
| 3744 | if (!match(&I, m_Not(m_OneUse(m_Xor(m_Value(X), m_Value(Y)))))) | ||||||||
| 3745 | return nullptr; | ||||||||
| 3746 | |||||||||
| 3747 | if (Instruction *NewXor = sinkNotIntoXor(I, X, Y, Builder)) | ||||||||
| 3748 | return NewXor; | ||||||||
| 3749 | |||||||||
| 3750 | auto hasCommonOperand = [](Value *A, Value *B, Value *C, Value *D) { | ||||||||
| 3751 | return A == C || A == D || B == C || B == D; | ||||||||
| 3752 | }; | ||||||||
| 3753 | |||||||||
| 3754 | Value *A, *B, *C, *D; | ||||||||
| 3755 | // Canonicalize ~((A & B) ^ (A | ?)) -> (A & B) | ~(A | ?) | ||||||||
| 3756 | // 4 commuted variants | ||||||||
| 3757 | if (match(X, m_And(m_Value(A), m_Value(B))) && | ||||||||
| 3758 | match(Y, m_Or(m_Value(C), m_Value(D))) && hasCommonOperand(A, B, C, D)) { | ||||||||
| 3759 | Value *NotY = Builder.CreateNot(Y); | ||||||||
| 3760 | return BinaryOperator::CreateOr(X, NotY); | ||||||||
| 3761 | }; | ||||||||
| 3762 | |||||||||
| 3763 | // Canonicalize ~((A | ?) ^ (A & B)) -> (A & B) | ~(A | ?) | ||||||||
| 3764 | // 4 commuted variants | ||||||||
| 3765 | if (match(Y, m_And(m_Value(A), m_Value(B))) && | ||||||||
| 3766 | match(X, m_Or(m_Value(C), m_Value(D))) && hasCommonOperand(A, B, C, D)) { | ||||||||
| 3767 | Value *NotX = Builder.CreateNot(X); | ||||||||
| 3768 | return BinaryOperator::CreateOr(Y, NotX); | ||||||||
| 3769 | }; | ||||||||
| 3770 | |||||||||
| 3771 | return nullptr; | ||||||||
| 3772 | } | ||||||||
| 3773 | |||||||||
| 3774 | /// Canonicalize a shifty way to code absolute value to the more common pattern | ||||||||
| 3775 | /// that uses negation and select. | ||||||||
| 3776 | static Instruction *canonicalizeAbs(BinaryOperator &Xor, | ||||||||
| 3777 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 3778 | assert(Xor.getOpcode() == Instruction::Xor && "Expected an xor instruction.")(static_cast <bool> (Xor.getOpcode() == Instruction::Xor && "Expected an xor instruction.") ? void (0) : __assert_fail ("Xor.getOpcode() == Instruction::Xor && \"Expected an xor instruction.\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 3778 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3779 | |||||||||
| 3780 | // There are 4 potential commuted variants. Move the 'ashr' candidate to Op1. | ||||||||
| 3781 | // We're relying on the fact that we only do this transform when the shift has | ||||||||
| 3782 | // exactly 2 uses and the add has exactly 1 use (otherwise, we might increase | ||||||||
| 3783 | // instructions). | ||||||||
| 3784 | Value *Op0 = Xor.getOperand(0), *Op1 = Xor.getOperand(1); | ||||||||
| 3785 | if (Op0->hasNUses(2)) | ||||||||
| 3786 | std::swap(Op0, Op1); | ||||||||
| 3787 | |||||||||
| 3788 | Type *Ty = Xor.getType(); | ||||||||
| 3789 | Value *A; | ||||||||
| 3790 | const APInt *ShAmt; | ||||||||
| 3791 | if (match(Op1, m_AShr(m_Value(A), m_APInt(ShAmt))) && | ||||||||
| 3792 | Op1->hasNUses(2) && *ShAmt == Ty->getScalarSizeInBits() - 1 && | ||||||||
| 3793 | match(Op0, m_OneUse(m_c_Add(m_Specific(A), m_Specific(Op1))))) { | ||||||||
| 3794 | // Op1 = ashr i32 A, 31 ; smear the sign bit | ||||||||
| 3795 | // xor (add A, Op1), Op1 ; add -1 and flip bits if negative | ||||||||
| 3796 | // --> (A < 0) ? -A : A | ||||||||
| 3797 | Value *IsNeg = Builder.CreateIsNeg(A); | ||||||||
| 3798 | // Copy the nuw/nsw flags from the add to the negate. | ||||||||
| 3799 | auto *Add = cast<BinaryOperator>(Op0); | ||||||||
| 3800 | Value *NegA = Builder.CreateNeg(A, "", Add->hasNoUnsignedWrap(), | ||||||||
| 3801 | Add->hasNoSignedWrap()); | ||||||||
| 3802 | return SelectInst::Create(IsNeg, NegA, A); | ||||||||
| 3803 | } | ||||||||
| 3804 | return nullptr; | ||||||||
| 3805 | } | ||||||||
| 3806 | |||||||||
| 3807 | // Transform | ||||||||
| 3808 | // z = ~(x &/| y) | ||||||||
| 3809 | // into: | ||||||||
| 3810 | // z = ((~x) |/& (~y)) | ||||||||
| 3811 | // iff both x and y are free to invert and all uses of z can be freely updated. | ||||||||
| 3812 | bool InstCombinerImpl::sinkNotIntoLogicalOp(Instruction &I) { | ||||||||
| 3813 | Value *Op0, *Op1; | ||||||||
| 3814 | if (!match(&I, m_LogicalOp(m_Value(Op0), m_Value(Op1)))) | ||||||||
| 3815 | return false; | ||||||||
| 3816 | |||||||||
| 3817 | // If this logic op has not been simplified yet, just bail out and let that | ||||||||
| 3818 | // happen first. Otherwise, the code below may wrongly invert. | ||||||||
| 3819 | if (Op0 == Op1) | ||||||||
| 3820 | return false; | ||||||||
| 3821 | |||||||||
| 3822 | Instruction::BinaryOps NewOpc = | ||||||||
| 3823 | match(&I, m_LogicalAnd()) ? Instruction::Or : Instruction::And; | ||||||||
| 3824 | bool IsBinaryOp = isa<BinaryOperator>(I); | ||||||||
| 3825 | |||||||||
| 3826 | // Can our users be adapted? | ||||||||
| 3827 | if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr)) | ||||||||
| 3828 | return false; | ||||||||
| 3829 | |||||||||
| 3830 | // And can the operands be adapted? | ||||||||
| 3831 | for (Value *Op : {Op0, Op1}) | ||||||||
| 3832 | if (!(InstCombiner::isFreeToInvert(Op, /*WillInvertAllUses=*/true) && | ||||||||
| 3833 | (match(Op, m_ImmConstant()) || | ||||||||
| 3834 | (isa<Instruction>(Op) && | ||||||||
| 3835 | InstCombiner::canFreelyInvertAllUsersOf(cast<Instruction>(Op), | ||||||||
| 3836 | /*IgnoredUser=*/&I))))) | ||||||||
| 3837 | return false; | ||||||||
| 3838 | |||||||||
| 3839 | for (Value **Op : {&Op0, &Op1}) { | ||||||||
| 3840 | Value *NotOp; | ||||||||
| 3841 | if (auto *C = dyn_cast<Constant>(*Op)) { | ||||||||
| 3842 | NotOp = ConstantExpr::getNot(C); | ||||||||
| 3843 | } else { | ||||||||
| 3844 | Builder.SetInsertPoint( | ||||||||
| 3845 | &*cast<Instruction>(*Op)->getInsertionPointAfterDef()); | ||||||||
| 3846 | NotOp = Builder.CreateNot(*Op, (*Op)->getName() + ".not"); | ||||||||
| 3847 | (*Op)->replaceUsesWithIf( | ||||||||
| 3848 | NotOp, [NotOp](Use &U) { return U.getUser() != NotOp; }); | ||||||||
| 3849 | freelyInvertAllUsersOf(NotOp, /*IgnoredUser=*/&I); | ||||||||
| 3850 | } | ||||||||
| 3851 | *Op = NotOp; | ||||||||
| 3852 | } | ||||||||
| 3853 | |||||||||
| 3854 | Builder.SetInsertPoint(I.getInsertionPointAfterDef()); | ||||||||
| 3855 | Value *NewLogicOp; | ||||||||
| 3856 | if (IsBinaryOp) | ||||||||
| 3857 | NewLogicOp = Builder.CreateBinOp(NewOpc, Op0, Op1, I.getName() + ".not"); | ||||||||
| 3858 | else | ||||||||
| 3859 | NewLogicOp = | ||||||||
| 3860 | Builder.CreateLogicalOp(NewOpc, Op0, Op1, I.getName() + ".not"); | ||||||||
| 3861 | |||||||||
| 3862 | replaceInstUsesWith(I, NewLogicOp); | ||||||||
| 3863 | // We can not just create an outer `not`, it will most likely be immediately | ||||||||
| 3864 | // folded back, reconstructing our initial pattern, and causing an | ||||||||
| 3865 | // infinite combine loop, so immediately manually fold it away. | ||||||||
| 3866 | freelyInvertAllUsersOf(NewLogicOp); | ||||||||
| 3867 | return true; | ||||||||
| 3868 | } | ||||||||
| 3869 | |||||||||
| 3870 | // Transform | ||||||||
| 3871 | // z = (~x) &/| y | ||||||||
| 3872 | // into: | ||||||||
| 3873 | // z = ~(x |/& (~y)) | ||||||||
| 3874 | // iff y is free to invert and all uses of z can be freely updated. | ||||||||
| 3875 | bool InstCombinerImpl::sinkNotIntoOtherHandOfLogicalOp(Instruction &I) { | ||||||||
| 3876 | Value *Op0, *Op1; | ||||||||
| 3877 | if (!match(&I, m_LogicalOp(m_Value(Op0), m_Value(Op1)))) | ||||||||
| 3878 | return false; | ||||||||
| 3879 | Instruction::BinaryOps NewOpc = | ||||||||
| 3880 | match(&I, m_LogicalAnd()) ? Instruction::Or : Instruction::And; | ||||||||
| 3881 | bool IsBinaryOp = isa<BinaryOperator>(I); | ||||||||
| 3882 | |||||||||
| 3883 | Value *NotOp0 = nullptr; | ||||||||
| 3884 | Value *NotOp1 = nullptr; | ||||||||
| 3885 | Value **OpToInvert = nullptr; | ||||||||
| 3886 | if (match(Op0, m_Not(m_Value(NotOp0))) && | ||||||||
| 3887 | InstCombiner::isFreeToInvert(Op1, /*WillInvertAllUses=*/true) && | ||||||||
| 3888 | (match(Op1, m_ImmConstant()) || | ||||||||
| 3889 | (isa<Instruction>(Op1) && | ||||||||
| 3890 | InstCombiner::canFreelyInvertAllUsersOf(cast<Instruction>(Op1), | ||||||||
| 3891 | /*IgnoredUser=*/&I)))) { | ||||||||
| 3892 | Op0 = NotOp0; | ||||||||
| 3893 | OpToInvert = &Op1; | ||||||||
| 3894 | } else if (match(Op1, m_Not(m_Value(NotOp1))) && | ||||||||
| 3895 | InstCombiner::isFreeToInvert(Op0, /*WillInvertAllUses=*/true) && | ||||||||
| 3896 | (match(Op0, m_ImmConstant()) || | ||||||||
| 3897 | (isa<Instruction>(Op0) && | ||||||||
| 3898 | InstCombiner::canFreelyInvertAllUsersOf(cast<Instruction>(Op0), | ||||||||
| 3899 | /*IgnoredUser=*/&I)))) { | ||||||||
| 3900 | Op1 = NotOp1; | ||||||||
| 3901 | OpToInvert = &Op0; | ||||||||
| 3902 | } else | ||||||||
| 3903 | return false; | ||||||||
| 3904 | |||||||||
| 3905 | // And can our users be adapted? | ||||||||
| 3906 | if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr)) | ||||||||
| 3907 | return false; | ||||||||
| 3908 | |||||||||
| 3909 | if (auto *C = dyn_cast<Constant>(*OpToInvert)) { | ||||||||
| 3910 | *OpToInvert = ConstantExpr::getNot(C); | ||||||||
| 3911 | } else { | ||||||||
| 3912 | Builder.SetInsertPoint( | ||||||||
| 3913 | &*cast<Instruction>(*OpToInvert)->getInsertionPointAfterDef()); | ||||||||
| 3914 | Value *NotOpToInvert = | ||||||||
| 3915 | Builder.CreateNot(*OpToInvert, (*OpToInvert)->getName() + ".not"); | ||||||||
| 3916 | (*OpToInvert)->replaceUsesWithIf(NotOpToInvert, [NotOpToInvert](Use &U) { | ||||||||
| 3917 | return U.getUser() != NotOpToInvert; | ||||||||
| 3918 | }); | ||||||||
| 3919 | freelyInvertAllUsersOf(NotOpToInvert, /*IgnoredUser=*/&I); | ||||||||
| 3920 | *OpToInvert = NotOpToInvert; | ||||||||
| 3921 | } | ||||||||
| 3922 | |||||||||
| 3923 | Builder.SetInsertPoint(&*I.getInsertionPointAfterDef()); | ||||||||
| 3924 | Value *NewBinOp; | ||||||||
| 3925 | if (IsBinaryOp) | ||||||||
| 3926 | NewBinOp = Builder.CreateBinOp(NewOpc, Op0, Op1, I.getName() + ".not"); | ||||||||
| 3927 | else | ||||||||
| 3928 | NewBinOp = Builder.CreateLogicalOp(NewOpc, Op0, Op1, I.getName() + ".not"); | ||||||||
| 3929 | replaceInstUsesWith(I, NewBinOp); | ||||||||
| 3930 | // We can not just create an outer `not`, it will most likely be immediately | ||||||||
| 3931 | // folded back, reconstructing our initial pattern, and causing an | ||||||||
| 3932 | // infinite combine loop, so immediately manually fold it away. | ||||||||
| 3933 | freelyInvertAllUsersOf(NewBinOp); | ||||||||
| 3934 | return true; | ||||||||
| 3935 | } | ||||||||
| 3936 | |||||||||
| 3937 | Instruction *InstCombinerImpl::foldNot(BinaryOperator &I) { | ||||||||
| 3938 | Value *NotOp; | ||||||||
| 3939 | if (!match(&I, m_Not(m_Value(NotOp)))) | ||||||||
| 3940 | return nullptr; | ||||||||
| 3941 | |||||||||
| 3942 | // Apply DeMorgan's Law for 'nand' / 'nor' logic with an inverted operand. | ||||||||
| 3943 | // We must eliminate the and/or (one-use) for these transforms to not increase | ||||||||
| 3944 | // the instruction count. | ||||||||
| 3945 | // | ||||||||
| 3946 | // ~(~X & Y) --> (X | ~Y) | ||||||||
| 3947 | // ~(Y & ~X) --> (X | ~Y) | ||||||||
| 3948 | // | ||||||||
| 3949 | // Note: The logical matches do not check for the commuted patterns because | ||||||||
| 3950 | // those are handled via SimplifySelectsFeedingBinaryOp(). | ||||||||
| 3951 | Type *Ty = I.getType(); | ||||||||
| 3952 | Value *X, *Y; | ||||||||
| 3953 | if (match(NotOp, m_OneUse(m_c_And(m_Not(m_Value(X)), m_Value(Y))))) { | ||||||||
| 3954 | Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not"); | ||||||||
| 3955 | return BinaryOperator::CreateOr(X, NotY); | ||||||||
| 3956 | } | ||||||||
| 3957 | if (match(NotOp, m_OneUse(m_LogicalAnd(m_Not(m_Value(X)), m_Value(Y))))) { | ||||||||
| 3958 | Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not"); | ||||||||
| 3959 | return SelectInst::Create(X, ConstantInt::getTrue(Ty), NotY); | ||||||||
| 3960 | } | ||||||||
| 3961 | |||||||||
| 3962 | // ~(~X | Y) --> (X & ~Y) | ||||||||
| 3963 | // ~(Y | ~X) --> (X & ~Y) | ||||||||
| 3964 | if (match(NotOp, m_OneUse(m_c_Or(m_Not(m_Value(X)), m_Value(Y))))) { | ||||||||
| 3965 | Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not"); | ||||||||
| 3966 | return BinaryOperator::CreateAnd(X, NotY); | ||||||||
| 3967 | } | ||||||||
| 3968 | if (match(NotOp, m_OneUse(m_LogicalOr(m_Not(m_Value(X)), m_Value(Y))))) { | ||||||||
| 3969 | Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not"); | ||||||||
| 3970 | return SelectInst::Create(X, NotY, ConstantInt::getFalse(Ty)); | ||||||||
| 3971 | } | ||||||||
| 3972 | |||||||||
| 3973 | // Is this a 'not' (~) fed by a binary operator? | ||||||||
| 3974 | BinaryOperator *NotVal; | ||||||||
| 3975 | if (match(NotOp, m_BinOp(NotVal))) { | ||||||||
| 3976 | // ~((-X) | Y) --> (X - 1) & (~Y) | ||||||||
| 3977 | if (match(NotVal, | ||||||||
| 3978 | m_OneUse(m_c_Or(m_OneUse(m_Neg(m_Value(X))), m_Value(Y))))) { | ||||||||
| 3979 | Value *DecX = Builder.CreateAdd(X, ConstantInt::getAllOnesValue(Ty)); | ||||||||
| 3980 | Value *NotY = Builder.CreateNot(Y); | ||||||||
| 3981 | return BinaryOperator::CreateAnd(DecX, NotY); | ||||||||
| 3982 | } | ||||||||
| 3983 | |||||||||
| 3984 | // ~(~X >>s Y) --> (X >>s Y) | ||||||||
| 3985 | if (match(NotVal, m_AShr(m_Not(m_Value(X)), m_Value(Y)))) | ||||||||
| 3986 | return BinaryOperator::CreateAShr(X, Y); | ||||||||
| 3987 | |||||||||
| 3988 | // Bit-hack form of a signbit test for iN type: | ||||||||
| 3989 | // ~(X >>s (N - 1)) --> sext i1 (X > -1) to iN | ||||||||
| 3990 | unsigned FullShift = Ty->getScalarSizeInBits() - 1; | ||||||||
| 3991 | if (match(NotVal, m_OneUse(m_AShr(m_Value(X), m_SpecificInt(FullShift))))) { | ||||||||
| 3992 | Value *IsNotNeg = Builder.CreateIsNotNeg(X, "isnotneg"); | ||||||||
| 3993 | return new SExtInst(IsNotNeg, Ty); | ||||||||
| 3994 | } | ||||||||
| 3995 | |||||||||
| 3996 | // If we are inverting a right-shifted constant, we may be able to eliminate | ||||||||
| 3997 | // the 'not' by inverting the constant and using the opposite shift type. | ||||||||
| 3998 | // Canonicalization rules ensure that only a negative constant uses 'ashr', | ||||||||
| 3999 | // but we must check that in case that transform has not fired yet. | ||||||||
| 4000 | |||||||||
| 4001 | // ~(C >>s Y) --> ~C >>u Y (when inverting the replicated sign bits) | ||||||||
| 4002 | Constant *C; | ||||||||
| 4003 | if (match(NotVal, m_AShr(m_Constant(C), m_Value(Y))) && | ||||||||
| 4004 | match(C, m_Negative())) { | ||||||||
| 4005 | // We matched a negative constant, so propagating undef is unsafe. | ||||||||
| 4006 | // Clamp undef elements to -1. | ||||||||
| 4007 | Type *EltTy = Ty->getScalarType(); | ||||||||
| 4008 | C = Constant::replaceUndefsWith(C, ConstantInt::getAllOnesValue(EltTy)); | ||||||||
| 4009 | return BinaryOperator::CreateLShr(ConstantExpr::getNot(C), Y); | ||||||||
| 4010 | } | ||||||||
| 4011 | |||||||||
| 4012 | // ~(C >>u Y) --> ~C >>s Y (when inverting the replicated sign bits) | ||||||||
| 4013 | if (match(NotVal, m_LShr(m_Constant(C), m_Value(Y))) && | ||||||||
| 4014 | match(C, m_NonNegative())) { | ||||||||
| 4015 | // We matched a non-negative constant, so propagating undef is unsafe. | ||||||||
| 4016 | // Clamp undef elements to 0. | ||||||||
| 4017 | Type *EltTy = Ty->getScalarType(); | ||||||||
| 4018 | C = Constant::replaceUndefsWith(C, ConstantInt::getNullValue(EltTy)); | ||||||||
| 4019 | return BinaryOperator::CreateAShr(ConstantExpr::getNot(C), Y); | ||||||||
| 4020 | } | ||||||||
| 4021 | |||||||||
| 4022 | // ~(X + C) --> ~C - X | ||||||||
| 4023 | if (match(NotVal, m_c_Add(m_Value(X), m_ImmConstant(C)))) | ||||||||
| 4024 | return BinaryOperator::CreateSub(ConstantExpr::getNot(C), X); | ||||||||
| 4025 | |||||||||
| 4026 | // ~(X - Y) --> ~X + Y | ||||||||
| 4027 | // FIXME: is it really beneficial to sink the `not` here? | ||||||||
| 4028 | if (match(NotVal, m_Sub(m_Value(X), m_Value(Y)))) | ||||||||
| 4029 | if (isa<Constant>(X) || NotVal->hasOneUse()) | ||||||||
| 4030 | return BinaryOperator::CreateAdd(Builder.CreateNot(X), Y); | ||||||||
| 4031 | |||||||||
| 4032 | // ~(~X + Y) --> X - Y | ||||||||
| 4033 | if (match(NotVal, m_c_Add(m_Not(m_Value(X)), m_Value(Y)))) | ||||||||
| 4034 | return BinaryOperator::CreateWithCopiedFlags(Instruction::Sub, X, Y, | ||||||||
| 4035 | NotVal); | ||||||||
| 4036 | } | ||||||||
| 4037 | |||||||||
| 4038 | // not (cmp A, B) = !cmp A, B | ||||||||
| 4039 | CmpInst::Predicate Pred; | ||||||||
| 4040 | if (match(NotOp, m_Cmp(Pred, m_Value(), m_Value())) && | ||||||||
| 4041 | (NotOp->hasOneUse() || | ||||||||
| 4042 | InstCombiner::canFreelyInvertAllUsersOf(cast<Instruction>(NotOp), | ||||||||
| 4043 | /*IgnoredUser=*/nullptr))) { | ||||||||
| 4044 | cast<CmpInst>(NotOp)->setPredicate(CmpInst::getInversePredicate(Pred)); | ||||||||
| 4045 | freelyInvertAllUsersOf(NotOp); | ||||||||
| 4046 | return &I; | ||||||||
| 4047 | } | ||||||||
| 4048 | |||||||||
| 4049 | // Move a 'not' ahead of casts of a bool to enable logic reduction: | ||||||||
| 4050 | // not (bitcast (sext i1 X)) --> bitcast (sext (not i1 X)) | ||||||||
| 4051 | if (match(NotOp, m_OneUse(m_BitCast(m_OneUse(m_SExt(m_Value(X)))))) && X->getType()->isIntOrIntVectorTy(1)) { | ||||||||
| 4052 | Type *SextTy = cast<BitCastOperator>(NotOp)->getSrcTy(); | ||||||||
| 4053 | Value *NotX = Builder.CreateNot(X); | ||||||||
| 4054 | Value *Sext = Builder.CreateSExt(NotX, SextTy); | ||||||||
| 4055 | return CastInst::CreateBitOrPointerCast(Sext, Ty); | ||||||||
| 4056 | } | ||||||||
| 4057 | |||||||||
| 4058 | if (auto *NotOpI = dyn_cast<Instruction>(NotOp)) | ||||||||
| 4059 | if (sinkNotIntoLogicalOp(*NotOpI)) | ||||||||
| 4060 | return &I; | ||||||||
| 4061 | |||||||||
| 4062 | // Eliminate a bitwise 'not' op of 'not' min/max by inverting the min/max: | ||||||||
| 4063 | // ~min(~X, ~Y) --> max(X, Y) | ||||||||
| 4064 | // ~max(~X, Y) --> min(X, ~Y) | ||||||||
| 4065 | auto *II = dyn_cast<IntrinsicInst>(NotOp); | ||||||||
| 4066 | if (II && II->hasOneUse()) { | ||||||||
| 4067 | if (match(NotOp, m_MaxOrMin(m_Value(X), m_Value(Y))) && | ||||||||
| 4068 | isFreeToInvert(X, X->hasOneUse()) && | ||||||||
| 4069 | isFreeToInvert(Y, Y->hasOneUse())) { | ||||||||
| 4070 | Intrinsic::ID InvID = getInverseMinMaxIntrinsic(II->getIntrinsicID()); | ||||||||
| 4071 | Value *NotX = Builder.CreateNot(X); | ||||||||
| 4072 | Value *NotY = Builder.CreateNot(Y); | ||||||||
| 4073 | Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, NotX, NotY); | ||||||||
| 4074 | return replaceInstUsesWith(I, InvMaxMin); | ||||||||
| 4075 | } | ||||||||
| 4076 | if (match(NotOp, m_c_MaxOrMin(m_Not(m_Value(X)), m_Value(Y)))) { | ||||||||
| 4077 | Intrinsic::ID InvID = getInverseMinMaxIntrinsic(II->getIntrinsicID()); | ||||||||
| 4078 | Value *NotY = Builder.CreateNot(Y); | ||||||||
| 4079 | Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, NotY); | ||||||||
| 4080 | return replaceInstUsesWith(I, InvMaxMin); | ||||||||
| 4081 | } | ||||||||
| 4082 | |||||||||
| 4083 | if (II->getIntrinsicID() == Intrinsic::is_fpclass) { | ||||||||
| 4084 | ConstantInt *ClassMask = cast<ConstantInt>(II->getArgOperand(1)); | ||||||||
| 4085 | II->setArgOperand( | ||||||||
| 4086 | 1, ConstantInt::get(ClassMask->getType(), | ||||||||
| 4087 | ~ClassMask->getZExtValue() & fcAllFlags)); | ||||||||
| 4088 | return replaceInstUsesWith(I, II); | ||||||||
| 4089 | } | ||||||||
| 4090 | } | ||||||||
| 4091 | |||||||||
| 4092 | if (NotOp->hasOneUse()) { | ||||||||
| 4093 | // Pull 'not' into operands of select if both operands are one-use compares | ||||||||
| 4094 | // or one is one-use compare and the other one is a constant. | ||||||||
| 4095 | // Inverting the predicates eliminates the 'not' operation. | ||||||||
| 4096 | // Example: | ||||||||
| 4097 | // not (select ?, (cmp TPred, ?, ?), (cmp FPred, ?, ?) --> | ||||||||
| 4098 | // select ?, (cmp InvTPred, ?, ?), (cmp InvFPred, ?, ?) | ||||||||
| 4099 | // not (select ?, (cmp TPred, ?, ?), true --> | ||||||||
| 4100 | // select ?, (cmp InvTPred, ?, ?), false | ||||||||
| 4101 | if (auto *Sel = dyn_cast<SelectInst>(NotOp)) { | ||||||||
| 4102 | Value *TV = Sel->getTrueValue(); | ||||||||
| 4103 | Value *FV = Sel->getFalseValue(); | ||||||||
| 4104 | auto *CmpT = dyn_cast<CmpInst>(TV); | ||||||||
| 4105 | auto *CmpF = dyn_cast<CmpInst>(FV); | ||||||||
| 4106 | bool InvertibleT = (CmpT && CmpT->hasOneUse()) || isa<Constant>(TV); | ||||||||
| 4107 | bool InvertibleF = (CmpF && CmpF->hasOneUse()) || isa<Constant>(FV); | ||||||||
| 4108 | if (InvertibleT && InvertibleF) { | ||||||||
| 4109 | if (CmpT) | ||||||||
| 4110 | CmpT->setPredicate(CmpT->getInversePredicate()); | ||||||||
| 4111 | else | ||||||||
| 4112 | Sel->setTrueValue(ConstantExpr::getNot(cast<Constant>(TV))); | ||||||||
| 4113 | if (CmpF) | ||||||||
| 4114 | CmpF->setPredicate(CmpF->getInversePredicate()); | ||||||||
| 4115 | else | ||||||||
| 4116 | Sel->setFalseValue(ConstantExpr::getNot(cast<Constant>(FV))); | ||||||||
| 4117 | return replaceInstUsesWith(I, Sel); | ||||||||
| 4118 | } | ||||||||
| 4119 | } | ||||||||
| 4120 | } | ||||||||
| 4121 | |||||||||
| 4122 | if (Instruction *NewXor = foldNotXor(I, Builder)) | ||||||||
| 4123 | return NewXor; | ||||||||
| 4124 | |||||||||
| 4125 | return nullptr; | ||||||||
| 4126 | } | ||||||||
| 4127 | |||||||||
| 4128 | // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches | ||||||||
| 4129 | // here. We should standardize that construct where it is needed or choose some | ||||||||
| 4130 | // other way to ensure that commutated variants of patterns are not missed. | ||||||||
| 4131 | Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) { | ||||||||
| 4132 | if (Value *V = simplifyXorInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 4133 | SQ.getWithInstruction(&I))) | ||||||||
| 4134 | return replaceInstUsesWith(I, V); | ||||||||
| 4135 | |||||||||
| 4136 | if (SimplifyAssociativeOrCommutative(I)) | ||||||||
| 4137 | return &I; | ||||||||
| 4138 | |||||||||
| 4139 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 4140 | return X; | ||||||||
| 4141 | |||||||||
| 4142 | if (Instruction *Phi = foldBinopWithPhiOperands(I)) | ||||||||
| 4143 | return Phi; | ||||||||
| 4144 | |||||||||
| 4145 | if (Instruction *NewXor = foldXorToXor(I, Builder)) | ||||||||
| 4146 | return NewXor; | ||||||||
| 4147 | |||||||||
| 4148 | // (A&B)^(A&C) -> A&(B^C) etc | ||||||||
| 4149 | if (Value *V = foldUsingDistributiveLaws(I)) | ||||||||
| 4150 | return replaceInstUsesWith(I, V); | ||||||||
| 4151 | |||||||||
| 4152 | // See if we can simplify any instructions used by the instruction whose sole | ||||||||
| 4153 | // purpose is to compute bits we don't care about. | ||||||||
| 4154 | if (SimplifyDemandedInstructionBits(I)) | ||||||||
| 4155 | return &I; | ||||||||
| 4156 | |||||||||
| 4157 | if (Value *V = SimplifyBSwap(I, Builder)) | ||||||||
| 4158 | return replaceInstUsesWith(I, V); | ||||||||
| 4159 | |||||||||
| 4160 | if (Instruction *R = foldNot(I)) | ||||||||
| 4161 | return R; | ||||||||
| 4162 | |||||||||
| 4163 | // Fold (X & M) ^ (Y & ~M) -> (X & M) | (Y & ~M) | ||||||||
| 4164 | // This it a special case in haveNoCommonBitsSet, but the computeKnownBits | ||||||||
| 4165 | // calls in there are unnecessary as SimplifyDemandedInstructionBits should | ||||||||
| 4166 | // have already taken care of those cases. | ||||||||
| 4167 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 4168 | Value *M; | ||||||||
| 4169 | if (match(&I, m_c_Xor(m_c_And(m_Not(m_Value(M)), m_Value()), | ||||||||
| 4170 | m_c_And(m_Deferred(M), m_Value())))) | ||||||||
| 4171 | return BinaryOperator::CreateOr(Op0, Op1); | ||||||||
| 4172 | |||||||||
| 4173 | if (Instruction *Xor = visitMaskedMerge(I, Builder)) | ||||||||
| 4174 | return Xor; | ||||||||
| 4175 | |||||||||
| 4176 | Value *X, *Y; | ||||||||
| 4177 | Constant *C1; | ||||||||
| 4178 | if (match(Op1, m_Constant(C1))) { | ||||||||
| 4179 | Constant *C2; | ||||||||
| 4180 | |||||||||
| 4181 | if (match(Op0, m_OneUse(m_Or(m_Value(X), m_ImmConstant(C2)))) && | ||||||||
| 4182 | match(C1, m_ImmConstant())) { | ||||||||
| 4183 | // (X | C2) ^ C1 --> (X & ~C2) ^ (C1^C2) | ||||||||
| 4184 | C2 = Constant::replaceUndefsWith( | ||||||||
| 4185 | C2, Constant::getAllOnesValue(C2->getType()->getScalarType())); | ||||||||
| 4186 | Value *And = Builder.CreateAnd( | ||||||||
| 4187 | X, Constant::mergeUndefsWith(ConstantExpr::getNot(C2), C1)); | ||||||||
| 4188 | return BinaryOperator::CreateXor( | ||||||||
| 4189 | And, Constant::mergeUndefsWith(ConstantExpr::getXor(C1, C2), C1)); | ||||||||
| 4190 | } | ||||||||
| 4191 | |||||||||
| 4192 | // Use DeMorgan and reassociation to eliminate a 'not' op. | ||||||||
| 4193 | if (match(Op0, m_OneUse(m_Or(m_Not(m_Value(X)), m_Constant(C2))))) { | ||||||||
| 4194 | // (~X | C2) ^ C1 --> ((X & ~C2) ^ -1) ^ C1 --> (X & ~C2) ^ ~C1 | ||||||||
| 4195 | Value *And = Builder.CreateAnd(X, ConstantExpr::getNot(C2)); | ||||||||
| 4196 | return BinaryOperator::CreateXor(And, ConstantExpr::getNot(C1)); | ||||||||
| 4197 | } | ||||||||
| 4198 | if (match(Op0, m_OneUse(m_And(m_Not(m_Value(X)), m_Constant(C2))))) { | ||||||||
| 4199 | // (~X & C2) ^ C1 --> ((X | ~C2) ^ -1) ^ C1 --> (X | ~C2) ^ ~C1 | ||||||||
| 4200 | Value *Or = Builder.CreateOr(X, ConstantExpr::getNot(C2)); | ||||||||
| 4201 | return BinaryOperator::CreateXor(Or, ConstantExpr::getNot(C1)); | ||||||||
| 4202 | } | ||||||||
| 4203 | |||||||||
| 4204 | // Convert xor ([trunc] (ashr X, BW-1)), C => | ||||||||
| 4205 | // select(X >s -1, C, ~C) | ||||||||
| 4206 | // The ashr creates "AllZeroOrAllOne's", which then optionally inverses the | ||||||||
| 4207 | // constant depending on whether this input is less than 0. | ||||||||
| 4208 | const APInt *CA; | ||||||||
| 4209 | if (match(Op0, m_OneUse(m_TruncOrSelf( | ||||||||
| 4210 | m_AShr(m_Value(X), m_APIntAllowUndef(CA))))) && | ||||||||
| 4211 | *CA == X->getType()->getScalarSizeInBits() - 1 && | ||||||||
| 4212 | !match(C1, m_AllOnes())) { | ||||||||
| 4213 | assert(!C1->isZeroValue() && "Unexpected xor with 0")(static_cast <bool> (!C1->isZeroValue() && "Unexpected xor with 0" ) ? void (0) : __assert_fail ("!C1->isZeroValue() && \"Unexpected xor with 0\"" , "llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp", 4213 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4214 | Value *IsNotNeg = Builder.CreateIsNotNeg(X); | ||||||||
| 4215 | return SelectInst::Create(IsNotNeg, Op1, Builder.CreateNot(Op1)); | ||||||||
| 4216 | } | ||||||||
| 4217 | } | ||||||||
| 4218 | |||||||||
| 4219 | Type *Ty = I.getType(); | ||||||||
| 4220 | { | ||||||||
| 4221 | const APInt *RHSC; | ||||||||
| 4222 | if (match(Op1, m_APInt(RHSC))) { | ||||||||
| 4223 | Value *X; | ||||||||
| 4224 | const APInt *C; | ||||||||
| 4225 | // (C - X) ^ signmaskC --> (C + signmaskC) - X | ||||||||
| 4226 | if (RHSC->isSignMask() && match(Op0, m_Sub(m_APInt(C), m_Value(X)))) | ||||||||
| 4227 | return BinaryOperator::CreateSub(ConstantInt::get(Ty, *C + *RHSC), X); | ||||||||
| 4228 | |||||||||
| 4229 | // (X + C) ^ signmaskC --> X + (C + signmaskC) | ||||||||
| 4230 | if (RHSC->isSignMask() && match(Op0, m_Add(m_Value(X), m_APInt(C)))) | ||||||||
| 4231 | return BinaryOperator::CreateAdd(X, ConstantInt::get(Ty, *C + *RHSC)); | ||||||||
| 4232 | |||||||||
| 4233 | // (X | C) ^ RHSC --> X ^ (C ^ RHSC) iff X & C == 0 | ||||||||
| 4234 | if (match(Op0, m_Or(m_Value(X), m_APInt(C))) && | ||||||||
| 4235 | MaskedValueIsZero(X, *C, 0, &I)) | ||||||||
| 4236 | return BinaryOperator::CreateXor(X, ConstantInt::get(Ty, *C ^ *RHSC)); | ||||||||
| 4237 | |||||||||
| 4238 | // When X is a power-of-two or zero and zero input is poison: | ||||||||
| 4239 | // ctlz(i32 X) ^ 31 --> cttz(X) | ||||||||
| 4240 | // cttz(i32 X) ^ 31 --> ctlz(X) | ||||||||
| 4241 | auto *II = dyn_cast<IntrinsicInst>(Op0); | ||||||||
| 4242 | if (II && II->hasOneUse() && *RHSC == Ty->getScalarSizeInBits() - 1) { | ||||||||
| 4243 | Intrinsic::ID IID = II->getIntrinsicID(); | ||||||||
| 4244 | if ((IID == Intrinsic::ctlz || IID == Intrinsic::cttz) && | ||||||||
| 4245 | match(II->getArgOperand(1), m_One()) && | ||||||||
| 4246 | isKnownToBeAPowerOfTwo(II->getArgOperand(0), /*OrZero */ true)) { | ||||||||
| 4247 | IID = (IID == Intrinsic::ctlz) ? Intrinsic::cttz : Intrinsic::ctlz; | ||||||||
| 4248 | Function *F = Intrinsic::getDeclaration(II->getModule(), IID, Ty); | ||||||||
| 4249 | return CallInst::Create(F, {II->getArgOperand(0), Builder.getTrue()}); | ||||||||
| 4250 | } | ||||||||
| 4251 | } | ||||||||
| 4252 | |||||||||
| 4253 | // If RHSC is inverting the remaining bits of shifted X, | ||||||||
| 4254 | // canonicalize to a 'not' before the shift to help SCEV and codegen: | ||||||||
| 4255 | // (X << C) ^ RHSC --> ~X << C | ||||||||
| 4256 | if (match(Op0, m_OneUse(m_Shl(m_Value(X), m_APInt(C)))) && | ||||||||
| 4257 | *RHSC == APInt::getAllOnes(Ty->getScalarSizeInBits()).shl(*C)) { | ||||||||
| 4258 | Value *NotX = Builder.CreateNot(X); | ||||||||
| 4259 | return BinaryOperator::CreateShl(NotX, ConstantInt::get(Ty, *C)); | ||||||||
| 4260 | } | ||||||||
| 4261 | // (X >>u C) ^ RHSC --> ~X >>u C | ||||||||
| 4262 | if (match(Op0, m_OneUse(m_LShr(m_Value(X), m_APInt(C)))) && | ||||||||
| 4263 | *RHSC == APInt::getAllOnes(Ty->getScalarSizeInBits()).lshr(*C)) { | ||||||||
| 4264 | Value *NotX = Builder.CreateNot(X); | ||||||||
| 4265 | return BinaryOperator::CreateLShr(NotX, ConstantInt::get(Ty, *C)); | ||||||||
| 4266 | } | ||||||||
| 4267 | // TODO: We could handle 'ashr' here as well. That would be matching | ||||||||
| 4268 | // a 'not' op and moving it before the shift. Doing that requires | ||||||||
| 4269 | // preventing the inverse fold in canShiftBinOpWithConstantRHS(). | ||||||||
| 4270 | } | ||||||||
| 4271 | } | ||||||||
| 4272 | |||||||||
| 4273 | // FIXME: This should not be limited to scalar (pull into APInt match above). | ||||||||
| 4274 | { | ||||||||
| 4275 | Value *X; | ||||||||
| 4276 | ConstantInt *C1, *C2, *C3; | ||||||||
| 4277 | // ((X^C1) >> C2) ^ C3 -> (X>>C2) ^ ((C1>>C2)^C3) | ||||||||
| 4278 | if (match(Op1, m_ConstantInt(C3)) && | ||||||||
| 4279 | match(Op0, m_LShr(m_Xor(m_Value(X), m_ConstantInt(C1)), | ||||||||
| 4280 | m_ConstantInt(C2))) && | ||||||||
| 4281 | Op0->hasOneUse()) { | ||||||||
| 4282 | // fold (C1 >> C2) ^ C3 | ||||||||
| 4283 | APInt FoldConst = C1->getValue().lshr(C2->getValue()); | ||||||||
| 4284 | FoldConst ^= C3->getValue(); | ||||||||
| 4285 | // Prepare the two operands. | ||||||||
| 4286 | auto *Opnd0 = Builder.CreateLShr(X, C2); | ||||||||
| 4287 | Opnd0->takeName(Op0); | ||||||||
| 4288 | return BinaryOperator::CreateXor(Opnd0, ConstantInt::get(Ty, FoldConst)); | ||||||||
| 4289 | } | ||||||||
| 4290 | } | ||||||||
| 4291 | |||||||||
| 4292 | if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I)) | ||||||||
| 4293 | return FoldedLogic; | ||||||||
| 4294 | |||||||||
| 4295 | // Y ^ (X | Y) --> X & ~Y | ||||||||
| 4296 | // Y ^ (Y | X) --> X & ~Y | ||||||||
| 4297 | if (match(Op1, m_OneUse(m_c_Or(m_Value(X), m_Specific(Op0))))) | ||||||||
| 4298 | return BinaryOperator::CreateAnd(X, Builder.CreateNot(Op0)); | ||||||||
| 4299 | // (X | Y) ^ Y --> X & ~Y | ||||||||
| 4300 | // (Y | X) ^ Y --> X & ~Y | ||||||||
| 4301 | if (match(Op0, m_OneUse(m_c_Or(m_Value(X), m_Specific(Op1))))) | ||||||||
| 4302 | return BinaryOperator::CreateAnd(X, Builder.CreateNot(Op1)); | ||||||||
| 4303 | |||||||||
| 4304 | // Y ^ (X & Y) --> ~X & Y | ||||||||
| 4305 | // Y ^ (Y & X) --> ~X & Y | ||||||||
| 4306 | if (match(Op1, m_OneUse(m_c_And(m_Value(X), m_Specific(Op0))))) | ||||||||
| 4307 | return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(X)); | ||||||||
| 4308 | // (X & Y) ^ Y --> ~X & Y | ||||||||
| 4309 | // (Y & X) ^ Y --> ~X & Y | ||||||||
| 4310 | // Canonical form is (X & C) ^ C; don't touch that. | ||||||||
| 4311 | // TODO: A 'not' op is better for analysis and codegen, but demanded bits must | ||||||||
| 4312 | // be fixed to prefer that (otherwise we get infinite looping). | ||||||||
| 4313 | if (!match(Op1, m_Constant()) && | ||||||||
| 4314 | match(Op0, m_OneUse(m_c_And(m_Value(X), m_Specific(Op1))))) | ||||||||
| 4315 | return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(X)); | ||||||||
| 4316 | |||||||||
| 4317 | Value *A, *B, *C; | ||||||||
| 4318 | // (A ^ B) ^ (A | C) --> (~A & C) ^ B -- There are 4 commuted variants. | ||||||||
| 4319 | if (match(&I, m_c_Xor(m_OneUse(m_Xor(m_Value(A), m_Value(B))), | ||||||||
| 4320 | m_OneUse(m_c_Or(m_Deferred(A), m_Value(C)))))) | ||||||||
| 4321 | return BinaryOperator::CreateXor( | ||||||||
| 4322 | Builder.CreateAnd(Builder.CreateNot(A), C), B); | ||||||||
| 4323 | |||||||||
| 4324 | // (A ^ B) ^ (B | C) --> (~B & C) ^ A -- There are 4 commuted variants. | ||||||||
| 4325 | if (match(&I, m_c_Xor(m_OneUse(m_Xor(m_Value(A), m_Value(B))), | ||||||||
| 4326 | m_OneUse(m_c_Or(m_Deferred(B), m_Value(C)))))) | ||||||||
| 4327 | return BinaryOperator::CreateXor( | ||||||||
| 4328 | Builder.CreateAnd(Builder.CreateNot(B), C), A); | ||||||||
| 4329 | |||||||||
| 4330 | // (A & B) ^ (A ^ B) -> (A | B) | ||||||||
| 4331 | if (match(Op0, m_And(m_Value(A), m_Value(B))) && | ||||||||
| 4332 | match(Op1, m_c_Xor(m_Specific(A), m_Specific(B)))) | ||||||||
| 4333 | return BinaryOperator::CreateOr(A, B); | ||||||||
| 4334 | // (A ^ B) ^ (A & B) -> (A | B) | ||||||||
| 4335 | if (match(Op0, m_Xor(m_Value(A), m_Value(B))) && | ||||||||
| 4336 | match(Op1, m_c_And(m_Specific(A), m_Specific(B)))) | ||||||||
| 4337 | return BinaryOperator::CreateOr(A, B); | ||||||||
| 4338 | |||||||||
| 4339 | // (A & ~B) ^ ~A -> ~(A & B) | ||||||||
| 4340 | // (~B & A) ^ ~A -> ~(A & B) | ||||||||
| 4341 | if (match(Op0, m_c_And(m_Value(A), m_Not(m_Value(B)))) && | ||||||||
| 4342 | match(Op1, m_Not(m_Specific(A)))) | ||||||||
| 4343 | return BinaryOperator::CreateNot(Builder.CreateAnd(A, B)); | ||||||||
| 4344 | |||||||||
| 4345 | // (~A & B) ^ A --> A | B -- There are 4 commuted variants. | ||||||||
| 4346 | if (match(&I, m_c_Xor(m_c_And(m_Not(m_Value(A)), m_Value(B)), m_Deferred(A)))) | ||||||||
| 4347 | return BinaryOperator::CreateOr(A, B); | ||||||||
| 4348 | |||||||||
| 4349 | // (~A | B) ^ A --> ~(A & B) | ||||||||
| 4350 | if (match(Op0, m_OneUse(m_c_Or(m_Not(m_Specific(Op1)), m_Value(B))))) | ||||||||
| 4351 | return BinaryOperator::CreateNot(Builder.CreateAnd(Op1, B)); | ||||||||
| 4352 | |||||||||
| 4353 | // A ^ (~A | B) --> ~(A & B) | ||||||||
| 4354 | if (match(Op1, m_OneUse(m_c_Or(m_Not(m_Specific(Op0)), m_Value(B))))) | ||||||||
| 4355 | return BinaryOperator::CreateNot(Builder.CreateAnd(Op0, B)); | ||||||||
| 4356 | |||||||||
| 4357 | // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants. | ||||||||
| 4358 | // TODO: Loosen one-use restriction if common operand is a constant. | ||||||||
| 4359 | Value *D; | ||||||||
| 4360 | if (match(Op0, m_OneUse(m_Or(m_Value(A), m_Value(B)))) && | ||||||||
| 4361 | match(Op1, m_OneUse(m_Or(m_Value(C), m_Value(D))))) { | ||||||||
| 4362 | if (B == C || B == D) | ||||||||
| 4363 | std::swap(A, B); | ||||||||
| 4364 | if (A == C) | ||||||||
| 4365 | std::swap(C, D); | ||||||||
| 4366 | if (A == D) { | ||||||||
| 4367 | Value *NotA = Builder.CreateNot(A); | ||||||||
| 4368 | return BinaryOperator::CreateAnd(Builder.CreateXor(B, C), NotA); | ||||||||
| 4369 | } | ||||||||
| 4370 | } | ||||||||
| 4371 | |||||||||
| 4372 | // (A & B) ^ (A | C) --> A ? ~B : C -- There are 4 commuted variants. | ||||||||
| 4373 | if (I.getType()->isIntOrIntVectorTy(1) && | ||||||||
| 4374 | match(Op0, m_OneUse(m_LogicalAnd(m_Value(A), m_Value(B)))) && | ||||||||
| 4375 | match(Op1, m_OneUse(m_LogicalOr(m_Value(C), m_Value(D))))) { | ||||||||
| 4376 | bool NeedFreeze = isa<SelectInst>(Op0) && isa<SelectInst>(Op1) && B == D; | ||||||||
| 4377 | if (B == C || B == D) | ||||||||
| 4378 | std::swap(A, B); | ||||||||
| 4379 | if (A == C) | ||||||||
| 4380 | std::swap(C, D); | ||||||||
| 4381 | if (A == D) { | ||||||||
| 4382 | if (NeedFreeze) | ||||||||
| 4383 | A = Builder.CreateFreeze(A); | ||||||||
| 4384 | Value *NotB = Builder.CreateNot(B); | ||||||||
| 4385 | return SelectInst::Create(A, NotB, C); | ||||||||
| 4386 | } | ||||||||
| 4387 | } | ||||||||
| 4388 | |||||||||
| 4389 | if (auto *LHS = dyn_cast<ICmpInst>(I.getOperand(0))) | ||||||||
| 4390 | if (auto *RHS = dyn_cast<ICmpInst>(I.getOperand(1))) | ||||||||
| 4391 | if (Value *V = foldXorOfICmps(LHS, RHS, I)) | ||||||||
| 4392 | return replaceInstUsesWith(I, V); | ||||||||
| 4393 | |||||||||
| 4394 | if (Instruction *CastedXor = foldCastedBitwiseLogic(I)) | ||||||||
| 4395 | return CastedXor; | ||||||||
| 4396 | |||||||||
| 4397 | if (Instruction *Abs = canonicalizeAbs(I, Builder)) | ||||||||
| 4398 | return Abs; | ||||||||
| 4399 | |||||||||
| 4400 | // Otherwise, if all else failed, try to hoist the xor-by-constant: | ||||||||
| 4401 | // (X ^ C) ^ Y --> (X ^ Y) ^ C | ||||||||
| 4402 | // Just like we do in other places, we completely avoid the fold | ||||||||
| 4403 | // for constantexprs, at least to avoid endless combine loop. | ||||||||
| 4404 | if (match(&I, m_c_Xor(m_OneUse(m_Xor(m_CombineAnd(m_Value(X), | ||||||||
| 4405 | m_Unless(m_ConstantExpr())), | ||||||||
| 4406 | m_ImmConstant(C1))), | ||||||||
| 4407 | m_Value(Y)))) | ||||||||
| 4408 | return BinaryOperator::CreateXor(Builder.CreateXor(X, Y), C1); | ||||||||
| 4409 | |||||||||
| 4410 | if (Instruction *R = reassociateForUses(I, Builder)) | ||||||||
| 4411 | return R; | ||||||||
| 4412 | |||||||||
| 4413 | if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder)) | ||||||||
| 4414 | return Canonicalized; | ||||||||
| 4415 | |||||||||
| 4416 | if (Instruction *Folded = foldLogicOfIsFPClass(I, Op0, Op1)) | ||||||||
| 4417 | return Folded; | ||||||||
| 4418 | |||||||||
| 4419 | if (Instruction *Folded = canonicalizeConditionalNegationViaMathToSelect(I)) | ||||||||
| 4420 | return Folded; | ||||||||
| 4421 | |||||||||
| 4422 | return nullptr; | ||||||||
| 4423 | } |
| 1 | //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // This file provides a simple and efficient mechanism for performing general | |||
| 10 | // tree-based pattern matches on the LLVM IR. The power of these routines is | |||
| 11 | // that it allows you to write concise patterns that are expressive and easy to | |||
| 12 | // understand. The other major advantage of this is that it allows you to | |||
| 13 | // trivially capture/bind elements in the pattern to variables. For example, | |||
| 14 | // you can do something like this: | |||
| 15 | // | |||
| 16 | // Value *Exp = ... | |||
| 17 | // Value *X, *Y; ConstantInt *C1, *C2; // (X & C1) | (Y & C2) | |||
| 18 | // if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)), | |||
| 19 | // m_And(m_Value(Y), m_ConstantInt(C2))))) { | |||
| 20 | // ... Pattern is matched and variables are bound ... | |||
| 21 | // } | |||
| 22 | // | |||
| 23 | // This is primarily useful to things like the instruction combiner, but can | |||
| 24 | // also be useful for static analysis tools or code generators. | |||
| 25 | // | |||
| 26 | //===----------------------------------------------------------------------===// | |||
| 27 | ||||
| 28 | #ifndef LLVM_IR_PATTERNMATCH_H | |||
| 29 | #define LLVM_IR_PATTERNMATCH_H | |||
| 30 | ||||
| 31 | #include "llvm/ADT/APFloat.h" | |||
| 32 | #include "llvm/ADT/APInt.h" | |||
| 33 | #include "llvm/IR/Constant.h" | |||
| 34 | #include "llvm/IR/Constants.h" | |||
| 35 | #include "llvm/IR/DataLayout.h" | |||
| 36 | #include "llvm/IR/InstrTypes.h" | |||
| 37 | #include "llvm/IR/Instruction.h" | |||
| 38 | #include "llvm/IR/Instructions.h" | |||
| 39 | #include "llvm/IR/IntrinsicInst.h" | |||
| 40 | #include "llvm/IR/Intrinsics.h" | |||
| 41 | #include "llvm/IR/Operator.h" | |||
| 42 | #include "llvm/IR/Value.h" | |||
| 43 | #include "llvm/Support/Casting.h" | |||
| 44 | #include <cstdint> | |||
| 45 | ||||
| 46 | namespace llvm { | |||
| 47 | namespace PatternMatch { | |||
| 48 | ||||
| 49 | template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) { | |||
| 50 | return const_cast<Pattern &>(P).match(V); | |||
| 51 | } | |||
| 52 | ||||
| 53 | template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) { | |||
| 54 | return const_cast<Pattern &>(P).match(Mask); | |||
| 55 | } | |||
| 56 | ||||
| 57 | template <typename SubPattern_t> struct OneUse_match { | |||
| 58 | SubPattern_t SubPattern; | |||
| 59 | ||||
| 60 | OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} | |||
| 61 | ||||
| 62 | template <typename OpTy> bool match(OpTy *V) { | |||
| 63 | return V->hasOneUse() && SubPattern.match(V); | |||
| 64 | } | |||
| 65 | }; | |||
| 66 | ||||
| 67 | template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) { | |||
| 68 | return SubPattern; | |||
| 69 | } | |||
| 70 | ||||
| 71 | template <typename Class> struct class_match { | |||
| 72 | template <typename ITy> bool match(ITy *V) { return isa<Class>(V); } | |||
| 73 | }; | |||
| 74 | ||||
| 75 | /// Match an arbitrary value and ignore it. | |||
| 76 | inline class_match<Value> m_Value() { return class_match<Value>(); } | |||
| 77 | ||||
| 78 | /// Match an arbitrary unary operation and ignore it. | |||
| 79 | inline class_match<UnaryOperator> m_UnOp() { | |||
| 80 | return class_match<UnaryOperator>(); | |||
| 81 | } | |||
| 82 | ||||
| 83 | /// Match an arbitrary binary operation and ignore it. | |||
| 84 | inline class_match<BinaryOperator> m_BinOp() { | |||
| 85 | return class_match<BinaryOperator>(); | |||
| 86 | } | |||
| 87 | ||||
| 88 | /// Matches any compare instruction and ignore it. | |||
| 89 | inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); } | |||
| 90 | ||||
| 91 | struct undef_match { | |||
| 92 | static bool check(const Value *V) { | |||
| 93 | if (isa<UndefValue>(V)) | |||
| 94 | return true; | |||
| 95 | ||||
| 96 | const auto *CA = dyn_cast<ConstantAggregate>(V); | |||
| 97 | if (!CA) | |||
| 98 | return false; | |||
| 99 | ||||
| 100 | SmallPtrSet<const ConstantAggregate *, 8> Seen; | |||
| 101 | SmallVector<const ConstantAggregate *, 8> Worklist; | |||
| 102 | ||||
| 103 | // Either UndefValue, PoisonValue, or an aggregate that only contains | |||
| 104 | // these is accepted by matcher. | |||
| 105 | // CheckValue returns false if CA cannot satisfy this constraint. | |||
| 106 | auto CheckValue = [&](const ConstantAggregate *CA) { | |||
| 107 | for (const Value *Op : CA->operand_values()) { | |||
| 108 | if (isa<UndefValue>(Op)) | |||
| 109 | continue; | |||
| 110 | ||||
| 111 | const auto *CA = dyn_cast<ConstantAggregate>(Op); | |||
| 112 | if (!CA) | |||
| 113 | return false; | |||
| 114 | if (Seen.insert(CA).second) | |||
| 115 | Worklist.emplace_back(CA); | |||
| 116 | } | |||
| 117 | ||||
| 118 | return true; | |||
| 119 | }; | |||
| 120 | ||||
| 121 | if (!CheckValue(CA)) | |||
| 122 | return false; | |||
| 123 | ||||
| 124 | while (!Worklist.empty()) { | |||
| 125 | if (!CheckValue(Worklist.pop_back_val())) | |||
| 126 | return false; | |||
| 127 | } | |||
| 128 | return true; | |||
| 129 | } | |||
| 130 | template <typename ITy> bool match(ITy *V) { return check(V); } | |||
| 131 | }; | |||
| 132 | ||||
| 133 | /// Match an arbitrary undef constant. This matches poison as well. | |||
| 134 | /// If this is an aggregate and contains a non-aggregate element that is | |||
| 135 | /// neither undef nor poison, the aggregate is not matched. | |||
| 136 | inline auto m_Undef() { return undef_match(); } | |||
| 137 | ||||
| 138 | /// Match an arbitrary poison constant. | |||
| 139 | inline class_match<PoisonValue> m_Poison() { | |||
| 140 | return class_match<PoisonValue>(); | |||
| 141 | } | |||
| 142 | ||||
| 143 | /// Match an arbitrary Constant and ignore it. | |||
| 144 | inline class_match<Constant> m_Constant() { return class_match<Constant>(); } | |||
| 145 | ||||
| 146 | /// Match an arbitrary ConstantInt and ignore it. | |||
| 147 | inline class_match<ConstantInt> m_ConstantInt() { | |||
| 148 | return class_match<ConstantInt>(); | |||
| 149 | } | |||
| 150 | ||||
| 151 | /// Match an arbitrary ConstantFP and ignore it. | |||
| 152 | inline class_match<ConstantFP> m_ConstantFP() { | |||
| 153 | return class_match<ConstantFP>(); | |||
| 154 | } | |||
| 155 | ||||
| 156 | struct constantexpr_match { | |||
| 157 | template <typename ITy> bool match(ITy *V) { | |||
| 158 | auto *C = dyn_cast<Constant>(V); | |||
| 159 | return C && (isa<ConstantExpr>(C) || C->containsConstantExpression()); | |||
| 160 | } | |||
| 161 | }; | |||
| 162 | ||||
| 163 | /// Match a constant expression or a constant that contains a constant | |||
| 164 | /// expression. | |||
| 165 | inline constantexpr_match m_ConstantExpr() { return constantexpr_match(); } | |||
| 166 | ||||
| 167 | /// Match an arbitrary basic block value and ignore it. | |||
| 168 | inline class_match<BasicBlock> m_BasicBlock() { | |||
| 169 | return class_match<BasicBlock>(); | |||
| 170 | } | |||
| 171 | ||||
| 172 | /// Inverting matcher | |||
| 173 | template <typename Ty> struct match_unless { | |||
| 174 | Ty M; | |||
| 175 | ||||
| 176 | match_unless(const Ty &Matcher) : M(Matcher) {} | |||
| 177 | ||||
| 178 | template <typename ITy> bool match(ITy *V) { return !M.match(V); } | |||
| 179 | }; | |||
| 180 | ||||
| 181 | /// Match if the inner matcher does *NOT* match. | |||
| 182 | template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) { | |||
| 183 | return match_unless<Ty>(M); | |||
| 184 | } | |||
| 185 | ||||
| 186 | /// Matching combinators | |||
| 187 | template <typename LTy, typename RTy> struct match_combine_or { | |||
| 188 | LTy L; | |||
| 189 | RTy R; | |||
| 190 | ||||
| 191 | match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | |||
| 192 | ||||
| 193 | template <typename ITy> bool match(ITy *V) { | |||
| 194 | if (L.match(V)) | |||
| 195 | return true; | |||
| 196 | if (R.match(V)) | |||
| 197 | return true; | |||
| 198 | return false; | |||
| 199 | } | |||
| 200 | }; | |||
| 201 | ||||
| 202 | template <typename LTy, typename RTy> struct match_combine_and { | |||
| 203 | LTy L; | |||
| 204 | RTy R; | |||
| 205 | ||||
| 206 | match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | |||
| 207 | ||||
| 208 | template <typename ITy> bool match(ITy *V) { | |||
| 209 | if (L.match(V)) | |||
| 210 | if (R.match(V)) | |||
| 211 | return true; | |||
| 212 | return false; | |||
| 213 | } | |||
| 214 | }; | |||
| 215 | ||||
| 216 | /// Combine two pattern matchers matching L || R | |||
| 217 | template <typename LTy, typename RTy> | |||
| 218 | inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) { | |||
| 219 | return match_combine_or<LTy, RTy>(L, R); | |||
| 220 | } | |||
| 221 | ||||
| 222 | /// Combine two pattern matchers matching L && R | |||
| 223 | template <typename LTy, typename RTy> | |||
| 224 | inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) { | |||
| 225 | return match_combine_and<LTy, RTy>(L, R); | |||
| 226 | } | |||
| 227 | ||||
| 228 | struct apint_match { | |||
| 229 | const APInt *&Res; | |||
| 230 | bool AllowUndef; | |||
| 231 | ||||
| 232 | apint_match(const APInt *&Res, bool AllowUndef) | |||
| 233 | : Res(Res), AllowUndef(AllowUndef) {} | |||
| 234 | ||||
| 235 | template <typename ITy> bool match(ITy *V) { | |||
| 236 | if (auto *CI = dyn_cast<ConstantInt>(V)) { | |||
| 237 | Res = &CI->getValue(); | |||
| 238 | return true; | |||
| 239 | } | |||
| 240 | if (V->getType()->isVectorTy()) | |||
| ||||
| 241 | if (const auto *C = dyn_cast<Constant>(V)) | |||
| 242 | if (auto *CI = | |||
| 243 | dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndef))) { | |||
| 244 | Res = &CI->getValue(); | |||
| 245 | return true; | |||
| 246 | } | |||
| 247 | return false; | |||
| 248 | } | |||
| 249 | }; | |||
| 250 | // Either constexpr if or renaming ConstantFP::getValueAPF to | |||
| 251 | // ConstantFP::getValue is needed to do it via single template | |||
| 252 | // function for both apint/apfloat. | |||
| 253 | struct apfloat_match { | |||
| 254 | const APFloat *&Res; | |||
| 255 | bool AllowUndef; | |||
| 256 | ||||
| 257 | apfloat_match(const APFloat *&Res, bool AllowUndef) | |||
| 258 | : Res(Res), AllowUndef(AllowUndef) {} | |||
| 259 | ||||
| 260 | template <typename ITy> bool match(ITy *V) { | |||
| 261 | if (auto *CI = dyn_cast<ConstantFP>(V)) { | |||
| 262 | Res = &CI->getValueAPF(); | |||
| 263 | return true; | |||
| 264 | } | |||
| 265 | if (V->getType()->isVectorTy()) | |||
| 266 | if (const auto *C = dyn_cast<Constant>(V)) | |||
| 267 | if (auto *CI = | |||
| 268 | dyn_cast_or_null<ConstantFP>(C->getSplatValue(AllowUndef))) { | |||
| 269 | Res = &CI->getValueAPF(); | |||
| 270 | return true; | |||
| 271 | } | |||
| 272 | return false; | |||
| 273 | } | |||
| 274 | }; | |||
| 275 | ||||
| 276 | /// Match a ConstantInt or splatted ConstantVector, binding the | |||
| 277 | /// specified pointer to the contained APInt. | |||
| 278 | inline apint_match m_APInt(const APInt *&Res) { | |||
| 279 | // Forbid undefs by default to maintain previous behavior. | |||
| 280 | return apint_match(Res, /* AllowUndef */ false); | |||
| 281 | } | |||
| 282 | ||||
| 283 | /// Match APInt while allowing undefs in splat vector constants. | |||
| 284 | inline apint_match m_APIntAllowUndef(const APInt *&Res) { | |||
| 285 | return apint_match(Res, /* AllowUndef */ true); | |||
| 286 | } | |||
| 287 | ||||
| 288 | /// Match APInt while forbidding undefs in splat vector constants. | |||
| 289 | inline apint_match m_APIntForbidUndef(const APInt *&Res) { | |||
| 290 | return apint_match(Res, /* AllowUndef */ false); | |||
| 291 | } | |||
| 292 | ||||
| 293 | /// Match a ConstantFP or splatted ConstantVector, binding the | |||
| 294 | /// specified pointer to the contained APFloat. | |||
| 295 | inline apfloat_match m_APFloat(const APFloat *&Res) { | |||
| 296 | // Forbid undefs by default to maintain previous behavior. | |||
| 297 | return apfloat_match(Res, /* AllowUndef */ false); | |||
| 298 | } | |||
| 299 | ||||
| 300 | /// Match APFloat while allowing undefs in splat vector constants. | |||
| 301 | inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) { | |||
| 302 | return apfloat_match(Res, /* AllowUndef */ true); | |||
| 303 | } | |||
| 304 | ||||
| 305 | /// Match APFloat while forbidding undefs in splat vector constants. | |||
| 306 | inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) { | |||
| 307 | return apfloat_match(Res, /* AllowUndef */ false); | |||
| 308 | } | |||
| 309 | ||||
| 310 | template <int64_t Val> struct constantint_match { | |||
| 311 | template <typename ITy> bool match(ITy *V) { | |||
| 312 | if (const auto *CI = dyn_cast<ConstantInt>(V)) { | |||
| 313 | const APInt &CIV = CI->getValue(); | |||
| 314 | if (Val >= 0) | |||
| 315 | return CIV == static_cast<uint64_t>(Val); | |||
| 316 | // If Val is negative, and CI is shorter than it, truncate to the right | |||
| 317 | // number of bits. If it is larger, then we have to sign extend. Just | |||
| 318 | // compare their negated values. | |||
| 319 | return -CIV == -Val; | |||
| 320 | } | |||
| 321 | return false; | |||
| 322 | } | |||
| 323 | }; | |||
| 324 | ||||
| 325 | /// Match a ConstantInt with a specific value. | |||
| 326 | template <int64_t Val> inline constantint_match<Val> m_ConstantInt() { | |||
| 327 | return constantint_match<Val>(); | |||
| 328 | } | |||
| 329 | ||||
| 330 | /// This helper class is used to match constant scalars, vector splats, | |||
| 331 | /// and fixed width vectors that satisfy a specified predicate. | |||
| 332 | /// For fixed width vector constants, undefined elements are ignored. | |||
| 333 | template <typename Predicate, typename ConstantVal> | |||
| 334 | struct cstval_pred_ty : public Predicate { | |||
| 335 | template <typename ITy> bool match(ITy *V) { | |||
| 336 | if (const auto *CV = dyn_cast<ConstantVal>(V)) | |||
| 337 | return this->isValue(CV->getValue()); | |||
| 338 | if (const auto *VTy = dyn_cast<VectorType>(V->getType())) { | |||
| 339 | if (const auto *C = dyn_cast<Constant>(V)) { | |||
| 340 | if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue())) | |||
| 341 | return this->isValue(CV->getValue()); | |||
| 342 | ||||
| 343 | // Number of elements of a scalable vector unknown at compile time | |||
| 344 | auto *FVTy = dyn_cast<FixedVectorType>(VTy); | |||
| 345 | if (!FVTy) | |||
| 346 | return false; | |||
| 347 | ||||
| 348 | // Non-splat vector constant: check each element for a match. | |||
| 349 | unsigned NumElts = FVTy->getNumElements(); | |||
| 350 | assert(NumElts != 0 && "Constant vector with no elements?")(static_cast <bool> (NumElts != 0 && "Constant vector with no elements?" ) ? void (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\"" , "llvm/include/llvm/IR/PatternMatch.h", 350, __extension__ __PRETTY_FUNCTION__ )); | |||
| 351 | bool HasNonUndefElements = false; | |||
| 352 | for (unsigned i = 0; i != NumElts; ++i) { | |||
| 353 | Constant *Elt = C->getAggregateElement(i); | |||
| 354 | if (!Elt) | |||
| 355 | return false; | |||
| 356 | if (isa<UndefValue>(Elt)) | |||
| 357 | continue; | |||
| 358 | auto *CV = dyn_cast<ConstantVal>(Elt); | |||
| 359 | if (!CV || !this->isValue(CV->getValue())) | |||
| 360 | return false; | |||
| 361 | HasNonUndefElements = true; | |||
| 362 | } | |||
| 363 | return HasNonUndefElements; | |||
| 364 | } | |||
| 365 | } | |||
| 366 | return false; | |||
| 367 | } | |||
| 368 | }; | |||
| 369 | ||||
| 370 | /// specialization of cstval_pred_ty for ConstantInt | |||
| 371 | template <typename Predicate> | |||
| 372 | using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>; | |||
| 373 | ||||
| 374 | /// specialization of cstval_pred_ty for ConstantFP | |||
| 375 | template <typename Predicate> | |||
| 376 | using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>; | |||
| 377 | ||||
| 378 | /// This helper class is used to match scalar and vector constants that | |||
| 379 | /// satisfy a specified predicate, and bind them to an APInt. | |||
| 380 | template <typename Predicate> struct api_pred_ty : public Predicate { | |||
| 381 | const APInt *&Res; | |||
| 382 | ||||
| 383 | api_pred_ty(const APInt *&R) : Res(R) {} | |||
| 384 | ||||
| 385 | template <typename ITy> bool match(ITy *V) { | |||
| 386 | if (const auto *CI = dyn_cast<ConstantInt>(V)) | |||
| 387 | if (this->isValue(CI->getValue())) { | |||
| 388 | Res = &CI->getValue(); | |||
| 389 | return true; | |||
| 390 | } | |||
| 391 | if (V->getType()->isVectorTy()) | |||
| 392 | if (const auto *C = dyn_cast<Constant>(V)) | |||
| 393 | if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue())) | |||
| 394 | if (this->isValue(CI->getValue())) { | |||
| 395 | Res = &CI->getValue(); | |||
| 396 | return true; | |||
| 397 | } | |||
| 398 | ||||
| 399 | return false; | |||
| 400 | } | |||
| 401 | }; | |||
| 402 | ||||
| 403 | /// This helper class is used to match scalar and vector constants that | |||
| 404 | /// satisfy a specified predicate, and bind them to an APFloat. | |||
| 405 | /// Undefs are allowed in splat vector constants. | |||
| 406 | template <typename Predicate> struct apf_pred_ty : public Predicate { | |||
| 407 | const APFloat *&Res; | |||
| 408 | ||||
| 409 | apf_pred_ty(const APFloat *&R) : Res(R) {} | |||
| 410 | ||||
| 411 | template <typename ITy> bool match(ITy *V) { | |||
| 412 | if (const auto *CI = dyn_cast<ConstantFP>(V)) | |||
| 413 | if (this->isValue(CI->getValue())) { | |||
| 414 | Res = &CI->getValue(); | |||
| 415 | return true; | |||
| 416 | } | |||
| 417 | if (V->getType()->isVectorTy()) | |||
| 418 | if (const auto *C = dyn_cast<Constant>(V)) | |||
| 419 | if (auto *CI = dyn_cast_or_null<ConstantFP>( | |||
| 420 | C->getSplatValue(/* AllowUndef */ true))) | |||
| 421 | if (this->isValue(CI->getValue())) { | |||
| 422 | Res = &CI->getValue(); | |||
| 423 | return true; | |||
| 424 | } | |||
| 425 | ||||
| 426 | return false; | |||
| 427 | } | |||
| 428 | }; | |||
| 429 | ||||
| 430 | /////////////////////////////////////////////////////////////////////////////// | |||
| 431 | // | |||
| 432 | // Encapsulate constant value queries for use in templated predicate matchers. | |||
| 433 | // This allows checking if constants match using compound predicates and works | |||
| 434 | // with vector constants, possibly with relaxed constraints. For example, ignore | |||
| 435 | // undef values. | |||
| 436 | // | |||
| 437 | /////////////////////////////////////////////////////////////////////////////// | |||
| 438 | ||||
| 439 | struct is_any_apint { | |||
| 440 | bool isValue(const APInt &C) { return true; } | |||
| 441 | }; | |||
| 442 | /// Match an integer or vector with any integral constant. | |||
| 443 | /// For vectors, this includes constants with undefined elements. | |||
| 444 | inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { | |||
| 445 | return cst_pred_ty<is_any_apint>(); | |||
| 446 | } | |||
| 447 | ||||
| 448 | struct is_all_ones { | |||
| 449 | bool isValue(const APInt &C) { return C.isAllOnes(); } | |||
| 450 | }; | |||
| 451 | /// Match an integer or vector with all bits set. | |||
| 452 | /// For vectors, this includes constants with undefined elements. | |||
| 453 | inline cst_pred_ty<is_all_ones> m_AllOnes() { | |||
| 454 | return cst_pred_ty<is_all_ones>(); | |||
| 455 | } | |||
| 456 | ||||
| 457 | struct is_maxsignedvalue { | |||
| 458 | bool isValue(const APInt &C) { return C.isMaxSignedValue(); } | |||
| 459 | }; | |||
| 460 | /// Match an integer or vector with values having all bits except for the high | |||
| 461 | /// bit set (0x7f...). | |||
| 462 | /// For vectors, this includes constants with undefined elements. | |||
| 463 | inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() { | |||
| 464 | return cst_pred_ty<is_maxsignedvalue>(); | |||
| 465 | } | |||
| 466 | inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) { | |||
| 467 | return V; | |||
| 468 | } | |||
| 469 | ||||
| 470 | struct is_negative { | |||
| 471 | bool isValue(const APInt &C) { return C.isNegative(); } | |||
| 472 | }; | |||
| 473 | /// Match an integer or vector of negative values. | |||
| 474 | /// For vectors, this includes constants with undefined elements. | |||
| 475 | inline cst_pred_ty<is_negative> m_Negative() { | |||
| 476 | return cst_pred_ty<is_negative>(); | |||
| 477 | } | |||
| 478 | inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { return V; } | |||
| 479 | ||||
| 480 | struct is_nonnegative { | |||
| 481 | bool isValue(const APInt &C) { return C.isNonNegative(); } | |||
| 482 | }; | |||
| 483 | /// Match an integer or vector of non-negative values. | |||
| 484 | /// For vectors, this includes constants with undefined elements. | |||
| 485 | inline cst_pred_ty<is_nonnegative> m_NonNegative() { | |||
| 486 | return cst_pred_ty<is_nonnegative>(); | |||
| 487 | } | |||
| 488 | inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { return V; } | |||
| 489 | ||||
| 490 | struct is_strictlypositive { | |||
| 491 | bool isValue(const APInt &C) { return C.isStrictlyPositive(); } | |||
| 492 | }; | |||
| 493 | /// Match an integer or vector of strictly positive values. | |||
| 494 | /// For vectors, this includes constants with undefined elements. | |||
| 495 | inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() { | |||
| 496 | return cst_pred_ty<is_strictlypositive>(); | |||
| 497 | } | |||
| 498 | inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) { | |||
| 499 | return V; | |||
| 500 | } | |||
| 501 | ||||
| 502 | struct is_nonpositive { | |||
| 503 | bool isValue(const APInt &C) { return C.isNonPositive(); } | |||
| 504 | }; | |||
| 505 | /// Match an integer or vector of non-positive values. | |||
| 506 | /// For vectors, this includes constants with undefined elements. | |||
| 507 | inline cst_pred_ty<is_nonpositive> m_NonPositive() { | |||
| 508 | return cst_pred_ty<is_nonpositive>(); | |||
| 509 | } | |||
| 510 | inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } | |||
| 511 | ||||
| 512 | struct is_one { | |||
| 513 | bool isValue(const APInt &C) { return C.isOne(); } | |||
| 514 | }; | |||
| 515 | /// Match an integer 1 or a vector with all elements equal to 1. | |||
| 516 | /// For vectors, this includes constants with undefined elements. | |||
| 517 | inline cst_pred_ty<is_one> m_One() { return cst_pred_ty<is_one>(); } | |||
| 518 | ||||
| 519 | struct is_zero_int { | |||
| 520 | bool isValue(const APInt &C) { return C.isZero(); } | |||
| 521 | }; | |||
| 522 | /// Match an integer 0 or a vector with all elements equal to 0. | |||
| 523 | /// For vectors, this includes constants with undefined elements. | |||
| 524 | inline cst_pred_ty<is_zero_int> m_ZeroInt() { | |||
| 525 | return cst_pred_ty<is_zero_int>(); | |||
| 526 | } | |||
| 527 | ||||
| 528 | struct is_zero { | |||
| 529 | template <typename ITy> bool match(ITy *V) { | |||
| 530 | auto *C = dyn_cast<Constant>(V); | |||
| 531 | // FIXME: this should be able to do something for scalable vectors | |||
| 532 | return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C)); | |||
| 533 | } | |||
| 534 | }; | |||
| 535 | /// Match any null constant or a vector with all elements equal to 0. | |||
| 536 | /// For vectors, this includes constants with undefined elements. | |||
| 537 | inline is_zero m_Zero() { return is_zero(); } | |||
| 538 | ||||
| 539 | struct is_power2 { | |||
| 540 | bool isValue(const APInt &C) { return C.isPowerOf2(); } | |||
| 541 | }; | |||
| 542 | /// Match an integer or vector power-of-2. | |||
| 543 | /// For vectors, this includes constants with undefined elements. | |||
| 544 | inline cst_pred_ty<is_power2> m_Power2() { return cst_pred_ty<is_power2>(); } | |||
| 545 | inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { return V; } | |||
| 546 | ||||
| 547 | struct is_negated_power2 { | |||
| 548 | bool isValue(const APInt &C) { return C.isNegatedPowerOf2(); } | |||
| 549 | }; | |||
| 550 | /// Match a integer or vector negated power-of-2. | |||
| 551 | /// For vectors, this includes constants with undefined elements. | |||
| 552 | inline cst_pred_ty<is_negated_power2> m_NegatedPower2() { | |||
| 553 | return cst_pred_ty<is_negated_power2>(); | |||
| 554 | } | |||
| 555 | inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) { | |||
| 556 | return V; | |||
| 557 | } | |||
| 558 | ||||
| 559 | struct is_power2_or_zero { | |||
| 560 | bool isValue(const APInt &C) { return !C || C.isPowerOf2(); } | |||
| 561 | }; | |||
| 562 | /// Match an integer or vector of 0 or power-of-2 values. | |||
| 563 | /// For vectors, this includes constants with undefined elements. | |||
| 564 | inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() { | |||
| 565 | return cst_pred_ty<is_power2_or_zero>(); | |||
| 566 | } | |||
| 567 | inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) { | |||
| 568 | return V; | |||
| 569 | } | |||
| 570 | ||||
| 571 | struct is_sign_mask { | |||
| 572 | bool isValue(const APInt &C) { return C.isSignMask(); } | |||
| 573 | }; | |||
| 574 | /// Match an integer or vector with only the sign bit(s) set. | |||
| 575 | /// For vectors, this includes constants with undefined elements. | |||
| 576 | inline cst_pred_ty<is_sign_mask> m_SignMask() { | |||
| 577 | return cst_pred_ty<is_sign_mask>(); | |||
| 578 | } | |||
| 579 | ||||
| 580 | struct is_lowbit_mask { | |||
| 581 | bool isValue(const APInt &C) { return C.isMask(); } | |||
| 582 | }; | |||
| 583 | /// Match an integer or vector with only the low bit(s) set. | |||
| 584 | /// For vectors, this includes constants with undefined elements. | |||
| 585 | inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { | |||
| 586 | return cst_pred_ty<is_lowbit_mask>(); | |||
| 587 | } | |||
| 588 | inline api_pred_ty<is_lowbit_mask> m_LowBitMask(const APInt *&V) { return V; } | |||
| 589 | ||||
| 590 | struct icmp_pred_with_threshold { | |||
| 591 | ICmpInst::Predicate Pred; | |||
| 592 | const APInt *Thr; | |||
| 593 | bool isValue(const APInt &C) { return ICmpInst::compare(C, *Thr, Pred); } | |||
| 594 | }; | |||
| 595 | /// Match an integer or vector with every element comparing 'pred' (eg/ne/...) | |||
| 596 | /// to Threshold. For vectors, this includes constants with undefined elements. | |||
| 597 | inline cst_pred_ty<icmp_pred_with_threshold> | |||
| 598 | m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) { | |||
| 599 | cst_pred_ty<icmp_pred_with_threshold> P; | |||
| 600 | P.Pred = Predicate; | |||
| 601 | P.Thr = &Threshold; | |||
| 602 | return P; | |||
| 603 | } | |||
| 604 | ||||
| 605 | struct is_nan { | |||
| 606 | bool isValue(const APFloat &C) { return C.isNaN(); } | |||
| 607 | }; | |||
| 608 | /// Match an arbitrary NaN constant. This includes quiet and signalling nans. | |||
| 609 | /// For vectors, this includes constants with undefined elements. | |||
| 610 | inline cstfp_pred_ty<is_nan> m_NaN() { return cstfp_pred_ty<is_nan>(); } | |||
| 611 | ||||
| 612 | struct is_nonnan { | |||
| 613 | bool isValue(const APFloat &C) { return !C.isNaN(); } | |||
| 614 | }; | |||
| 615 | /// Match a non-NaN FP constant. | |||
| 616 | /// For vectors, this includes constants with undefined elements. | |||
| 617 | inline cstfp_pred_ty<is_nonnan> m_NonNaN() { | |||
| 618 | return cstfp_pred_ty<is_nonnan>(); | |||
| 619 | } | |||
| 620 | ||||
| 621 | struct is_inf { | |||
| 622 | bool isValue(const APFloat &C) { return C.isInfinity(); } | |||
| 623 | }; | |||
| 624 | /// Match a positive or negative infinity FP constant. | |||
| 625 | /// For vectors, this includes constants with undefined elements. | |||
| 626 | inline cstfp_pred_ty<is_inf> m_Inf() { return cstfp_pred_ty<is_inf>(); } | |||
| 627 | ||||
| 628 | struct is_noninf { | |||
| 629 | bool isValue(const APFloat &C) { return !C.isInfinity(); } | |||
| 630 | }; | |||
| 631 | /// Match a non-infinity FP constant, i.e. finite or NaN. | |||
| 632 | /// For vectors, this includes constants with undefined elements. | |||
| 633 | inline cstfp_pred_ty<is_noninf> m_NonInf() { | |||
| 634 | return cstfp_pred_ty<is_noninf>(); | |||
| 635 | } | |||
| 636 | ||||
| 637 | struct is_finite { | |||
| 638 | bool isValue(const APFloat &C) { return C.isFinite(); } | |||
| 639 | }; | |||
| 640 | /// Match a finite FP constant, i.e. not infinity or NaN. | |||
| 641 | /// For vectors, this includes constants with undefined elements. | |||
| 642 | inline cstfp_pred_ty<is_finite> m_Finite() { | |||
| 643 | return cstfp_pred_ty<is_finite>(); | |||
| 644 | } | |||
| 645 | inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; } | |||
| 646 | ||||
| 647 | struct is_finitenonzero { | |||
| 648 | bool isValue(const APFloat &C) { return C.isFiniteNonZero(); } | |||
| 649 | }; | |||
| 650 | /// Match a finite non-zero FP constant. | |||
| 651 | /// For vectors, this includes constants with undefined elements. | |||
| 652 | inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() { | |||
| 653 | return cstfp_pred_ty<is_finitenonzero>(); | |||
| 654 | } | |||
| 655 | inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) { | |||
| 656 | return V; | |||
| 657 | } | |||
| 658 | ||||
| 659 | struct is_any_zero_fp { | |||
| 660 | bool isValue(const APFloat &C) { return C.isZero(); } | |||
| 661 | }; | |||
| 662 | /// Match a floating-point negative zero or positive zero. | |||
| 663 | /// For vectors, this includes constants with undefined elements. | |||
| 664 | inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() { | |||
| 665 | return cstfp_pred_ty<is_any_zero_fp>(); | |||
| 666 | } | |||
| 667 | ||||
| 668 | struct is_pos_zero_fp { | |||
| 669 | bool isValue(const APFloat &C) { return C.isPosZero(); } | |||
| 670 | }; | |||
| 671 | /// Match a floating-point positive zero. | |||
| 672 | /// For vectors, this includes constants with undefined elements. | |||
| 673 | inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() { | |||
| 674 | return cstfp_pred_ty<is_pos_zero_fp>(); | |||
| 675 | } | |||
| 676 | ||||
| 677 | struct is_neg_zero_fp { | |||
| 678 | bool isValue(const APFloat &C) { return C.isNegZero(); } | |||
| 679 | }; | |||
| 680 | /// Match a floating-point negative zero. | |||
| 681 | /// For vectors, this includes constants with undefined elements. | |||
| 682 | inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() { | |||
| 683 | return cstfp_pred_ty<is_neg_zero_fp>(); | |||
| 684 | } | |||
| 685 | ||||
| 686 | struct is_non_zero_fp { | |||
| 687 | bool isValue(const APFloat &C) { return C.isNonZero(); } | |||
| 688 | }; | |||
| 689 | /// Match a floating-point non-zero. | |||
| 690 | /// For vectors, this includes constants with undefined elements. | |||
| 691 | inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() { | |||
| 692 | return cstfp_pred_ty<is_non_zero_fp>(); | |||
| 693 | } | |||
| 694 | ||||
| 695 | /////////////////////////////////////////////////////////////////////////////// | |||
| 696 | ||||
| 697 | template <typename Class> struct bind_ty { | |||
| 698 | Class *&VR; | |||
| 699 | ||||
| 700 | bind_ty(Class *&V) : VR(V) {} | |||
| 701 | ||||
| 702 | template <typename ITy> bool match(ITy *V) { | |||
| 703 | if (auto *CV = dyn_cast<Class>(V)) { | |||
| 704 | VR = CV; | |||
| 705 | return true; | |||
| 706 | } | |||
| 707 | return false; | |||
| 708 | } | |||
| 709 | }; | |||
| 710 | ||||
| 711 | /// Match a value, capturing it if we match. | |||
| 712 | inline bind_ty<Value> m_Value(Value *&V) { return V; } | |||
| 713 | inline bind_ty<const Value> m_Value(const Value *&V) { return V; } | |||
| 714 | ||||
| 715 | /// Match an instruction, capturing it if we match. | |||
| 716 | inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } | |||
| 717 | /// Match a unary operator, capturing it if we match. | |||
| 718 | inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; } | |||
| 719 | /// Match a binary operator, capturing it if we match. | |||
| 720 | inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } | |||
| 721 | /// Match a with overflow intrinsic, capturing it if we match. | |||
| 722 | inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { | |||
| 723 | return I; | |||
| 724 | } | |||
| 725 | inline bind_ty<const WithOverflowInst> | |||
| 726 | m_WithOverflowInst(const WithOverflowInst *&I) { | |||
| 727 | return I; | |||
| 728 | } | |||
| 729 | ||||
| 730 | /// Match a Constant, capturing the value if we match. | |||
| 731 | inline bind_ty<Constant> m_Constant(Constant *&C) { return C; } | |||
| 732 | ||||
| 733 | /// Match a ConstantInt, capturing the value if we match. | |||
| 734 | inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; } | |||
| 735 | ||||
| 736 | /// Match a ConstantFP, capturing the value if we match. | |||
| 737 | inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; } | |||
| 738 | ||||
| 739 | /// Match a ConstantExpr, capturing the value if we match. | |||
| 740 | inline bind_ty<ConstantExpr> m_ConstantExpr(ConstantExpr *&C) { return C; } | |||
| 741 | ||||
| 742 | /// Match a basic block value, capturing it if we match. | |||
| 743 | inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; } | |||
| 744 | inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) { | |||
| 745 | return V; | |||
| 746 | } | |||
| 747 | ||||
| 748 | /// Match an arbitrary immediate Constant and ignore it. | |||
| 749 | inline match_combine_and<class_match<Constant>, | |||
| 750 | match_unless<constantexpr_match>> | |||
| 751 | m_ImmConstant() { | |||
| 752 | return m_CombineAnd(m_Constant(), m_Unless(m_ConstantExpr())); | |||
| 753 | } | |||
| 754 | ||||
| 755 | /// Match an immediate Constant, capturing the value if we match. | |||
| 756 | inline match_combine_and<bind_ty<Constant>, | |||
| 757 | match_unless<constantexpr_match>> | |||
| 758 | m_ImmConstant(Constant *&C) { | |||
| 759 | return m_CombineAnd(m_Constant(C), m_Unless(m_ConstantExpr())); | |||
| 760 | } | |||
| 761 | ||||
| 762 | /// Match a specified Value*. | |||
| 763 | struct specificval_ty { | |||
| 764 | const Value *Val; | |||
| 765 | ||||
| 766 | specificval_ty(const Value *V) : Val(V) {} | |||
| 767 | ||||
| 768 | template <typename ITy> bool match(ITy *V) { return V == Val; } | |||
| 769 | }; | |||
| 770 | ||||
| 771 | /// Match if we have a specific specified value. | |||
| 772 | inline specificval_ty m_Specific(const Value *V) { return V; } | |||
| 773 | ||||
| 774 | /// Stores a reference to the Value *, not the Value * itself, | |||
| 775 | /// thus can be used in commutative matchers. | |||
| 776 | template <typename Class> struct deferredval_ty { | |||
| 777 | Class *const &Val; | |||
| 778 | ||||
| 779 | deferredval_ty(Class *const &V) : Val(V) {} | |||
| 780 | ||||
| 781 | template <typename ITy> bool match(ITy *const V) { return V == Val; } | |||
| 782 | }; | |||
| 783 | ||||
| 784 | /// Like m_Specific(), but works if the specific value to match is determined | |||
| 785 | /// as part of the same match() expression. For example: | |||
| 786 | /// m_Add(m_Value(X), m_Specific(X)) is incorrect, because m_Specific() will | |||
| 787 | /// bind X before the pattern match starts. | |||
| 788 | /// m_Add(m_Value(X), m_Deferred(X)) is correct, and will check against | |||
| 789 | /// whichever value m_Value(X) populated. | |||
| 790 | inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; } | |||
| 791 | inline deferredval_ty<const Value> m_Deferred(const Value *const &V) { | |||
| 792 | return V; | |||
| 793 | } | |||
| 794 | ||||
| 795 | /// Match a specified floating point value or vector of all elements of | |||
| 796 | /// that value. | |||
| 797 | struct specific_fpval { | |||
| 798 | double Val; | |||
| 799 | ||||
| 800 | specific_fpval(double V) : Val(V) {} | |||
| 801 | ||||
| 802 | template <typename ITy> bool match(ITy *V) { | |||
| 803 | if (const auto *CFP = dyn_cast<ConstantFP>(V)) | |||
| 804 | return CFP->isExactlyValue(Val); | |||
| 805 | if (V->getType()->isVectorTy()) | |||
| 806 | if (const auto *C = dyn_cast<Constant>(V)) | |||
| 807 | if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) | |||
| 808 | return CFP->isExactlyValue(Val); | |||
| 809 | return false; | |||
| 810 | } | |||
| 811 | }; | |||
| 812 | ||||
| 813 | /// Match a specific floating point value or vector with all elements | |||
| 814 | /// equal to the value. | |||
| 815 | inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); } | |||
| 816 | ||||
| 817 | /// Match a float 1.0 or vector with all elements equal to 1.0. | |||
| 818 | inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); } | |||
| 819 | ||||
| 820 | struct bind_const_intval_ty { | |||
| 821 | uint64_t &VR; | |||
| 822 | ||||
| 823 | bind_const_intval_ty(uint64_t &V) : VR(V) {} | |||
| 824 | ||||
| 825 | template <typename ITy> bool match(ITy *V) { | |||
| 826 | if (const auto *CV = dyn_cast<ConstantInt>(V)) | |||
| 827 | if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) { | |||
| 828 | VR = CV->getZExtValue(); | |||
| 829 | return true; | |||
| 830 | } | |||
| 831 | return false; | |||
| 832 | } | |||
| 833 | }; | |||
| 834 | ||||
| 835 | /// Match a specified integer value or vector of all elements of that | |||
| 836 | /// value. | |||
| 837 | template <bool AllowUndefs> struct specific_intval { | |||
| 838 | APInt Val; | |||
| 839 | ||||
| 840 | specific_intval(APInt V) : Val(std::move(V)) {} | |||
| 841 | ||||
| 842 | template <typename ITy> bool match(ITy *V) { | |||
| 843 | const auto *CI = dyn_cast<ConstantInt>(V); | |||
| 844 | if (!CI && V->getType()->isVectorTy()) | |||
| 845 | if (const auto *C = dyn_cast<Constant>(V)) | |||
| 846 | CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs)); | |||
| 847 | ||||
| 848 | return CI && APInt::isSameValue(CI->getValue(), Val); | |||
| 849 | } | |||
| 850 | }; | |||
| 851 | ||||
| 852 | /// Match a specific integer value or vector with all elements equal to | |||
| 853 | /// the value. | |||
| 854 | inline specific_intval<false> m_SpecificInt(APInt V) { | |||
| 855 | return specific_intval<false>(std::move(V)); | |||
| 856 | } | |||
| 857 | ||||
| 858 | inline specific_intval<false> m_SpecificInt(uint64_t V) { | |||
| 859 | return m_SpecificInt(APInt(64, V)); | |||
| 860 | } | |||
| 861 | ||||
| 862 | inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) { | |||
| 863 | return specific_intval<true>(std::move(V)); | |||
| 864 | } | |||
| 865 | ||||
| 866 | inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) { | |||
| 867 | return m_SpecificIntAllowUndef(APInt(64, V)); | |||
| 868 | } | |||
| 869 | ||||
| 870 | /// Match a ConstantInt and bind to its value. This does not match | |||
| 871 | /// ConstantInts wider than 64-bits. | |||
| 872 | inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } | |||
| 873 | ||||
| 874 | /// Match a specified basic block value. | |||
| 875 | struct specific_bbval { | |||
| 876 | BasicBlock *Val; | |||
| 877 | ||||
| 878 | specific_bbval(BasicBlock *Val) : Val(Val) {} | |||
| 879 | ||||
| 880 | template <typename ITy> bool match(ITy *V) { | |||
| 881 | const auto *BB = dyn_cast<BasicBlock>(V); | |||
| 882 | return BB && BB == Val; | |||
| 883 | } | |||
| 884 | }; | |||
| 885 | ||||
| 886 | /// Match a specific basic block value. | |||
| 887 | inline specific_bbval m_SpecificBB(BasicBlock *BB) { | |||
| 888 | return specific_bbval(BB); | |||
| 889 | } | |||
| 890 | ||||
| 891 | /// A commutative-friendly version of m_Specific(). | |||
| 892 | inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) { | |||
| 893 | return BB; | |||
| 894 | } | |||
| 895 | inline deferredval_ty<const BasicBlock> | |||
| 896 | m_Deferred(const BasicBlock *const &BB) { | |||
| 897 | return BB; | |||
| 898 | } | |||
| 899 | ||||
| 900 | //===----------------------------------------------------------------------===// | |||
| 901 | // Matcher for any binary operator. | |||
| 902 | // | |||
| 903 | template <typename LHS_t, typename RHS_t, bool Commutable = false> | |||
| 904 | struct AnyBinaryOp_match { | |||
| 905 | LHS_t L; | |||
| 906 | RHS_t R; | |||
| 907 | ||||
| 908 | // The evaluation order is always stable, regardless of Commutability. | |||
| 909 | // The LHS is always matched first. | |||
| 910 | AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | |||
| 911 | ||||
| 912 | template <typename OpTy> bool match(OpTy *V) { | |||
| 913 | if (auto *I = dyn_cast<BinaryOperator>(V)) | |||
| 914 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || | |||
| 915 | (Commutable && L.match(I->getOperand(1)) && | |||
| 916 | R.match(I->getOperand(0))); | |||
| 917 | return false; | |||
| 918 | } | |||
| 919 | }; | |||
| 920 | ||||
| 921 | template <typename LHS, typename RHS> | |||
| 922 | inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) { | |||
| 923 | return AnyBinaryOp_match<LHS, RHS>(L, R); | |||
| 924 | } | |||
| 925 | ||||
| 926 | //===----------------------------------------------------------------------===// | |||
| 927 | // Matcher for any unary operator. | |||
| 928 | // TODO fuse unary, binary matcher into n-ary matcher | |||
| 929 | // | |||
| 930 | template <typename OP_t> struct AnyUnaryOp_match { | |||
| 931 | OP_t X; | |||
| 932 | ||||
| 933 | AnyUnaryOp_match(const OP_t &X) : X(X) {} | |||
| 934 | ||||
| 935 | template <typename OpTy> bool match(OpTy *V) { | |||
| 936 | if (auto *I = dyn_cast<UnaryOperator>(V)) | |||
| 937 | return X.match(I->getOperand(0)); | |||
| 938 | return false; | |||
| 939 | } | |||
| 940 | }; | |||
| 941 | ||||
| 942 | template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) { | |||
| 943 | return AnyUnaryOp_match<OP_t>(X); | |||
| 944 | } | |||
| 945 | ||||
| 946 | //===----------------------------------------------------------------------===// | |||
| 947 | // Matchers for specific binary operators. | |||
| 948 | // | |||
| 949 | ||||
| 950 | template <typename LHS_t, typename RHS_t, unsigned Opcode, | |||
| 951 | bool Commutable = false> | |||
| 952 | struct BinaryOp_match { | |||
| 953 | LHS_t L; | |||
| 954 | RHS_t R; | |||
| 955 | ||||
| 956 | // The evaluation order is always stable, regardless of Commutability. | |||
| 957 | // The LHS is always matched first. | |||
| 958 | BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | |||
| 959 | ||||
| 960 | template <typename OpTy> inline bool match(unsigned Opc, OpTy *V) { | |||
| 961 | if (V->getValueID() == Value::InstructionVal + Opc) { | |||
| 962 | auto *I = cast<BinaryOperator>(V); | |||
| 963 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || | |||
| 964 | (Commutable && L.match(I->getOperand(1)) && | |||
| 965 | R.match(I->getOperand(0))); | |||
| 966 | } | |||
| 967 | if (auto *CE = dyn_cast<ConstantExpr>(V)) | |||
| 968 | return CE->getOpcode() == Opc && | |||
| 969 | ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) || | |||
| 970 | (Commutable && L.match(CE->getOperand(1)) && | |||
| 971 | R.match(CE->getOperand(0)))); | |||
| 972 | return false; | |||
| 973 | } | |||
| 974 | ||||
| 975 | template <typename OpTy> bool match(OpTy *V) { return match(Opcode, V); } | |||
| 976 | }; | |||
| 977 | ||||
| 978 | template <typename LHS, typename RHS> | |||
| 979 | inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L, | |||
| 980 | const RHS &R) { | |||
| 981 | return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R); | |||
| 982 | } | |||
| 983 | ||||
| 984 | template <typename LHS, typename RHS> | |||
| 985 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L, | |||
| 986 | const RHS &R) { | |||
| 987 | return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R); | |||
| 988 | } | |||
| 989 | ||||
| 990 | template <typename LHS, typename RHS> | |||
| 991 | inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L, | |||
| 992 | const RHS &R) { | |||
| 993 | return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R); | |||
| 994 | } | |||
| 995 | ||||
| 996 | template <typename LHS, typename RHS> | |||
| 997 | inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L, | |||
| 998 | const RHS &R) { | |||
| 999 | return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R); | |||
| 1000 | } | |||
| 1001 | ||||
| 1002 | template <typename Op_t> struct FNeg_match { | |||
| 1003 | Op_t X; | |||
| 1004 | ||||
| 1005 | FNeg_match(const Op_t &Op) : X(Op) {} | |||
| 1006 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1007 | auto *FPMO = dyn_cast<FPMathOperator>(V); | |||
| 1008 | if (!FPMO) | |||
| 1009 | return false; | |||
| 1010 | ||||
| 1011 | if (FPMO->getOpcode() == Instruction::FNeg) | |||
| 1012 | return X.match(FPMO->getOperand(0)); | |||
| 1013 | ||||
| 1014 | if (FPMO->getOpcode() == Instruction::FSub) { | |||
| 1015 | if (FPMO->hasNoSignedZeros()) { | |||
| 1016 | // With 'nsz', any zero goes. | |||
| 1017 | if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0))) | |||
| 1018 | return false; | |||
| 1019 | } else { | |||
| 1020 | // Without 'nsz', we need fsub -0.0, X exactly. | |||
| 1021 | if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0))) | |||
| 1022 | return false; | |||
| 1023 | } | |||
| 1024 | ||||
| 1025 | return X.match(FPMO->getOperand(1)); | |||
| 1026 | } | |||
| 1027 | ||||
| 1028 | return false; | |||
| 1029 | } | |||
| 1030 | }; | |||
| 1031 | ||||
| 1032 | /// Match 'fneg X' as 'fsub -0.0, X'. | |||
| 1033 | template <typename OpTy> inline FNeg_match<OpTy> m_FNeg(const OpTy &X) { | |||
| 1034 | return FNeg_match<OpTy>(X); | |||
| 1035 | } | |||
| 1036 | ||||
| 1037 | /// Match 'fneg X' as 'fsub +-0.0, X'. | |||
| 1038 | template <typename RHS> | |||
| 1039 | inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub> | |||
| 1040 | m_FNegNSZ(const RHS &X) { | |||
| 1041 | return m_FSub(m_AnyZeroFP(), X); | |||
| 1042 | } | |||
| 1043 | ||||
| 1044 | template <typename LHS, typename RHS> | |||
| 1045 | inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L, | |||
| 1046 | const RHS &R) { | |||
| 1047 | return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R); | |||
| 1048 | } | |||
| 1049 | ||||
| 1050 | template <typename LHS, typename RHS> | |||
| 1051 | inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L, | |||
| 1052 | const RHS &R) { | |||
| 1053 | return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R); | |||
| 1054 | } | |||
| 1055 | ||||
| 1056 | template <typename LHS, typename RHS> | |||
| 1057 | inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L, | |||
| 1058 | const RHS &R) { | |||
| 1059 | return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R); | |||
| 1060 | } | |||
| 1061 | ||||
| 1062 | template <typename LHS, typename RHS> | |||
| 1063 | inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L, | |||
| 1064 | const RHS &R) { | |||
| 1065 | return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R); | |||
| 1066 | } | |||
| 1067 | ||||
| 1068 | template <typename LHS, typename RHS> | |||
| 1069 | inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L, | |||
| 1070 | const RHS &R) { | |||
| 1071 | return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R); | |||
| 1072 | } | |||
| 1073 | ||||
| 1074 | template <typename LHS, typename RHS> | |||
| 1075 | inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L, | |||
| 1076 | const RHS &R) { | |||
| 1077 | return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R); | |||
| 1078 | } | |||
| 1079 | ||||
| 1080 | template <typename LHS, typename RHS> | |||
| 1081 | inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L, | |||
| 1082 | const RHS &R) { | |||
| 1083 | return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R); | |||
| 1084 | } | |||
| 1085 | ||||
| 1086 | template <typename LHS, typename RHS> | |||
| 1087 | inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L, | |||
| 1088 | const RHS &R) { | |||
| 1089 | return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R); | |||
| 1090 | } | |||
| 1091 | ||||
| 1092 | template <typename LHS, typename RHS> | |||
| 1093 | inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L, | |||
| 1094 | const RHS &R) { | |||
| 1095 | return BinaryOp_match<LHS, RHS, Instruction::And>(L, R); | |||
| 1096 | } | |||
| 1097 | ||||
| 1098 | template <typename LHS, typename RHS> | |||
| 1099 | inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L, | |||
| 1100 | const RHS &R) { | |||
| 1101 | return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R); | |||
| 1102 | } | |||
| 1103 | ||||
| 1104 | template <typename LHS, typename RHS> | |||
| 1105 | inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L, | |||
| 1106 | const RHS &R) { | |||
| 1107 | return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R); | |||
| 1108 | } | |||
| 1109 | ||||
| 1110 | template <typename LHS, typename RHS> | |||
| 1111 | inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L, | |||
| 1112 | const RHS &R) { | |||
| 1113 | return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R); | |||
| 1114 | } | |||
| 1115 | ||||
| 1116 | template <typename LHS, typename RHS> | |||
| 1117 | inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L, | |||
| 1118 | const RHS &R) { | |||
| 1119 | return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R); | |||
| 1120 | } | |||
| 1121 | ||||
| 1122 | template <typename LHS, typename RHS> | |||
| 1123 | inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L, | |||
| 1124 | const RHS &R) { | |||
| 1125 | return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R); | |||
| 1126 | } | |||
| 1127 | ||||
| 1128 | template <typename LHS_t, typename RHS_t, unsigned Opcode, | |||
| 1129 | unsigned WrapFlags = 0> | |||
| 1130 | struct OverflowingBinaryOp_match { | |||
| 1131 | LHS_t L; | |||
| 1132 | RHS_t R; | |||
| 1133 | ||||
| 1134 | OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) | |||
| 1135 | : L(LHS), R(RHS) {} | |||
| 1136 | ||||
| 1137 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1138 | if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) { | |||
| 1139 | if (Op->getOpcode() != Opcode) | |||
| 1140 | return false; | |||
| 1141 | if ((WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap) && | |||
| 1142 | !Op->hasNoUnsignedWrap()) | |||
| 1143 | return false; | |||
| 1144 | if ((WrapFlags & OverflowingBinaryOperator::NoSignedWrap) && | |||
| 1145 | !Op->hasNoSignedWrap()) | |||
| 1146 | return false; | |||
| 1147 | return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1)); | |||
| 1148 | } | |||
| 1149 | return false; | |||
| 1150 | } | |||
| 1151 | }; | |||
| 1152 | ||||
| 1153 | template <typename LHS, typename RHS> | |||
| 1154 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | |||
| 1155 | OverflowingBinaryOperator::NoSignedWrap> | |||
| 1156 | m_NSWAdd(const LHS &L, const RHS &R) { | |||
| 1157 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | |||
| 1158 | OverflowingBinaryOperator::NoSignedWrap>(L, | |||
| 1159 | R); | |||
| 1160 | } | |||
| 1161 | template <typename LHS, typename RHS> | |||
| 1162 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | |||
| 1163 | OverflowingBinaryOperator::NoSignedWrap> | |||
| 1164 | m_NSWSub(const LHS &L, const RHS &R) { | |||
| 1165 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | |||
| 1166 | OverflowingBinaryOperator::NoSignedWrap>(L, | |||
| 1167 | R); | |||
| 1168 | } | |||
| 1169 | template <typename LHS, typename RHS> | |||
| 1170 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | |||
| 1171 | OverflowingBinaryOperator::NoSignedWrap> | |||
| 1172 | m_NSWMul(const LHS &L, const RHS &R) { | |||
| 1173 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | |||
| 1174 | OverflowingBinaryOperator::NoSignedWrap>(L, | |||
| 1175 | R); | |||
| 1176 | } | |||
| 1177 | template <typename LHS, typename RHS> | |||
| 1178 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | |||
| 1179 | OverflowingBinaryOperator::NoSignedWrap> | |||
| 1180 | m_NSWShl(const LHS &L, const RHS &R) { | |||
| 1181 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | |||
| 1182 | OverflowingBinaryOperator::NoSignedWrap>(L, | |||
| 1183 | R); | |||
| 1184 | } | |||
| 1185 | ||||
| 1186 | template <typename LHS, typename RHS> | |||
| 1187 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | |||
| 1188 | OverflowingBinaryOperator::NoUnsignedWrap> | |||
| 1189 | m_NUWAdd(const LHS &L, const RHS &R) { | |||
| 1190 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | |||
| 1191 | OverflowingBinaryOperator::NoUnsignedWrap>( | |||
| 1192 | L, R); | |||
| 1193 | } | |||
| 1194 | template <typename LHS, typename RHS> | |||
| 1195 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | |||
| 1196 | OverflowingBinaryOperator::NoUnsignedWrap> | |||
| 1197 | m_NUWSub(const LHS &L, const RHS &R) { | |||
| 1198 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | |||
| 1199 | OverflowingBinaryOperator::NoUnsignedWrap>( | |||
| 1200 | L, R); | |||
| 1201 | } | |||
| 1202 | template <typename LHS, typename RHS> | |||
| 1203 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | |||
| 1204 | OverflowingBinaryOperator::NoUnsignedWrap> | |||
| 1205 | m_NUWMul(const LHS &L, const RHS &R) { | |||
| 1206 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | |||
| 1207 | OverflowingBinaryOperator::NoUnsignedWrap>( | |||
| 1208 | L, R); | |||
| 1209 | } | |||
| 1210 | template <typename LHS, typename RHS> | |||
| 1211 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | |||
| 1212 | OverflowingBinaryOperator::NoUnsignedWrap> | |||
| 1213 | m_NUWShl(const LHS &L, const RHS &R) { | |||
| 1214 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | |||
| 1215 | OverflowingBinaryOperator::NoUnsignedWrap>( | |||
| 1216 | L, R); | |||
| 1217 | } | |||
| 1218 | ||||
| 1219 | template <typename LHS_t, typename RHS_t, bool Commutable = false> | |||
| 1220 | struct SpecificBinaryOp_match | |||
| 1221 | : public BinaryOp_match<LHS_t, RHS_t, 0, Commutable> { | |||
| 1222 | unsigned Opcode; | |||
| 1223 | ||||
| 1224 | SpecificBinaryOp_match(unsigned Opcode, const LHS_t &LHS, const RHS_t &RHS) | |||
| 1225 | : BinaryOp_match<LHS_t, RHS_t, 0, Commutable>(LHS, RHS), Opcode(Opcode) {} | |||
| 1226 | ||||
| 1227 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1228 | return BinaryOp_match<LHS_t, RHS_t, 0, Commutable>::match(Opcode, V); | |||
| 1229 | } | |||
| 1230 | }; | |||
| 1231 | ||||
| 1232 | /// Matches a specific opcode. | |||
| 1233 | template <typename LHS, typename RHS> | |||
| 1234 | inline SpecificBinaryOp_match<LHS, RHS> m_BinOp(unsigned Opcode, const LHS &L, | |||
| 1235 | const RHS &R) { | |||
| 1236 | return SpecificBinaryOp_match<LHS, RHS>(Opcode, L, R); | |||
| 1237 | } | |||
| 1238 | ||||
| 1239 | //===----------------------------------------------------------------------===// | |||
| 1240 | // Class that matches a group of binary opcodes. | |||
| 1241 | // | |||
| 1242 | template <typename LHS_t, typename RHS_t, typename Predicate> | |||
| 1243 | struct BinOpPred_match : Predicate { | |||
| 1244 | LHS_t L; | |||
| 1245 | RHS_t R; | |||
| 1246 | ||||
| 1247 | BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | |||
| 1248 | ||||
| 1249 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1250 | if (auto *I = dyn_cast<Instruction>(V)) | |||
| 1251 | return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) && | |||
| 1252 | R.match(I->getOperand(1)); | |||
| 1253 | if (auto *CE = dyn_cast<ConstantExpr>(V)) | |||
| 1254 | return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) && | |||
| 1255 | R.match(CE->getOperand(1)); | |||
| 1256 | return false; | |||
| 1257 | } | |||
| 1258 | }; | |||
| 1259 | ||||
| 1260 | struct is_shift_op { | |||
| 1261 | bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); } | |||
| 1262 | }; | |||
| 1263 | ||||
| 1264 | struct is_right_shift_op { | |||
| 1265 | bool isOpType(unsigned Opcode) { | |||
| 1266 | return Opcode == Instruction::LShr || Opcode == Instruction::AShr; | |||
| 1267 | } | |||
| 1268 | }; | |||
| 1269 | ||||
| 1270 | struct is_logical_shift_op { | |||
| 1271 | bool isOpType(unsigned Opcode) { | |||
| 1272 | return Opcode == Instruction::LShr || Opcode == Instruction::Shl; | |||
| 1273 | } | |||
| 1274 | }; | |||
| 1275 | ||||
| 1276 | struct is_bitwiselogic_op { | |||
| 1277 | bool isOpType(unsigned Opcode) { | |||
| 1278 | return Instruction::isBitwiseLogicOp(Opcode); | |||
| 1279 | } | |||
| 1280 | }; | |||
| 1281 | ||||
| 1282 | struct is_idiv_op { | |||
| 1283 | bool isOpType(unsigned Opcode) { | |||
| 1284 | return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv; | |||
| 1285 | } | |||
| 1286 | }; | |||
| 1287 | ||||
| 1288 | struct is_irem_op { | |||
| 1289 | bool isOpType(unsigned Opcode) { | |||
| 1290 | return Opcode == Instruction::SRem || Opcode == Instruction::URem; | |||
| 1291 | } | |||
| 1292 | }; | |||
| 1293 | ||||
| 1294 | /// Matches shift operations. | |||
| 1295 | template <typename LHS, typename RHS> | |||
| 1296 | inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L, | |||
| 1297 | const RHS &R) { | |||
| 1298 | return BinOpPred_match<LHS, RHS, is_shift_op>(L, R); | |||
| 1299 | } | |||
| 1300 | ||||
| 1301 | /// Matches logical shift operations. | |||
| 1302 | template <typename LHS, typename RHS> | |||
| 1303 | inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L, | |||
| 1304 | const RHS &R) { | |||
| 1305 | return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R); | |||
| 1306 | } | |||
| 1307 | ||||
| 1308 | /// Matches logical shift operations. | |||
| 1309 | template <typename LHS, typename RHS> | |||
| 1310 | inline BinOpPred_match<LHS, RHS, is_logical_shift_op> | |||
| 1311 | m_LogicalShift(const LHS &L, const RHS &R) { | |||
| 1312 | return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R); | |||
| 1313 | } | |||
| 1314 | ||||
| 1315 | /// Matches bitwise logic operations. | |||
| 1316 | template <typename LHS, typename RHS> | |||
| 1317 | inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op> | |||
| 1318 | m_BitwiseLogic(const LHS &L, const RHS &R) { | |||
| 1319 | return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R); | |||
| 1320 | } | |||
| 1321 | ||||
| 1322 | /// Matches integer division operations. | |||
| 1323 | template <typename LHS, typename RHS> | |||
| 1324 | inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L, | |||
| 1325 | const RHS &R) { | |||
| 1326 | return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R); | |||
| 1327 | } | |||
| 1328 | ||||
| 1329 | /// Matches integer remainder operations. | |||
| 1330 | template <typename LHS, typename RHS> | |||
| 1331 | inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L, | |||
| 1332 | const RHS &R) { | |||
| 1333 | return BinOpPred_match<LHS, RHS, is_irem_op>(L, R); | |||
| 1334 | } | |||
| 1335 | ||||
| 1336 | //===----------------------------------------------------------------------===// | |||
| 1337 | // Class that matches exact binary ops. | |||
| 1338 | // | |||
| 1339 | template <typename SubPattern_t> struct Exact_match { | |||
| 1340 | SubPattern_t SubPattern; | |||
| 1341 | ||||
| 1342 | Exact_match(const SubPattern_t &SP) : SubPattern(SP) {} | |||
| 1343 | ||||
| 1344 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1345 | if (auto *PEO = dyn_cast<PossiblyExactOperator>(V)) | |||
| 1346 | return PEO->isExact() && SubPattern.match(V); | |||
| 1347 | return false; | |||
| 1348 | } | |||
| 1349 | }; | |||
| 1350 | ||||
| 1351 | template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) { | |||
| 1352 | return SubPattern; | |||
| 1353 | } | |||
| 1354 | ||||
| 1355 | //===----------------------------------------------------------------------===// | |||
| 1356 | // Matchers for CmpInst classes | |||
| 1357 | // | |||
| 1358 | ||||
| 1359 | template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy, | |||
| 1360 | bool Commutable = false> | |||
| 1361 | struct CmpClass_match { | |||
| 1362 | PredicateTy &Predicate; | |||
| 1363 | LHS_t L; | |||
| 1364 | RHS_t R; | |||
| 1365 | ||||
| 1366 | // The evaluation order is always stable, regardless of Commutability. | |||
| 1367 | // The LHS is always matched first. | |||
| 1368 | CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS) | |||
| 1369 | : Predicate(Pred), L(LHS), R(RHS) {} | |||
| 1370 | ||||
| 1371 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1372 | if (auto *I = dyn_cast<Class>(V)) { | |||
| 1373 | if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) { | |||
| 1374 | Predicate = I->getPredicate(); | |||
| 1375 | return true; | |||
| 1376 | } else if (Commutable && L.match(I->getOperand(1)) && | |||
| 1377 | R.match(I->getOperand(0))) { | |||
| 1378 | Predicate = I->getSwappedPredicate(); | |||
| 1379 | return true; | |||
| 1380 | } | |||
| 1381 | } | |||
| 1382 | return false; | |||
| 1383 | } | |||
| 1384 | }; | |||
| 1385 | ||||
| 1386 | template <typename LHS, typename RHS> | |||
| 1387 | inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate> | |||
| 1388 | m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | |||
| 1389 | return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R); | |||
| 1390 | } | |||
| 1391 | ||||
| 1392 | template <typename LHS, typename RHS> | |||
| 1393 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate> | |||
| 1394 | m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | |||
| 1395 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R); | |||
| 1396 | } | |||
| 1397 | ||||
| 1398 | template <typename LHS, typename RHS> | |||
| 1399 | inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate> | |||
| 1400 | m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | |||
| 1401 | return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R); | |||
| 1402 | } | |||
| 1403 | ||||
| 1404 | //===----------------------------------------------------------------------===// | |||
| 1405 | // Matchers for instructions with a given opcode and number of operands. | |||
| 1406 | // | |||
| 1407 | ||||
| 1408 | /// Matches instructions with Opcode and three operands. | |||
| 1409 | template <typename T0, unsigned Opcode> struct OneOps_match { | |||
| 1410 | T0 Op1; | |||
| 1411 | ||||
| 1412 | OneOps_match(const T0 &Op1) : Op1(Op1) {} | |||
| 1413 | ||||
| 1414 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1415 | if (V->getValueID() == Value::InstructionVal + Opcode) { | |||
| 1416 | auto *I = cast<Instruction>(V); | |||
| 1417 | return Op1.match(I->getOperand(0)); | |||
| 1418 | } | |||
| 1419 | return false; | |||
| 1420 | } | |||
| 1421 | }; | |||
| 1422 | ||||
| 1423 | /// Matches instructions with Opcode and three operands. | |||
| 1424 | template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match { | |||
| 1425 | T0 Op1; | |||
| 1426 | T1 Op2; | |||
| 1427 | ||||
| 1428 | TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {} | |||
| 1429 | ||||
| 1430 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1431 | if (V->getValueID() == Value::InstructionVal + Opcode) { | |||
| 1432 | auto *I = cast<Instruction>(V); | |||
| 1433 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)); | |||
| 1434 | } | |||
| 1435 | return false; | |||
| 1436 | } | |||
| 1437 | }; | |||
| 1438 | ||||
| 1439 | /// Matches instructions with Opcode and three operands. | |||
| 1440 | template <typename T0, typename T1, typename T2, unsigned Opcode> | |||
| 1441 | struct ThreeOps_match { | |||
| 1442 | T0 Op1; | |||
| 1443 | T1 Op2; | |||
| 1444 | T2 Op3; | |||
| 1445 | ||||
| 1446 | ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3) | |||
| 1447 | : Op1(Op1), Op2(Op2), Op3(Op3) {} | |||
| 1448 | ||||
| 1449 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1450 | if (V->getValueID() == Value::InstructionVal + Opcode) { | |||
| 1451 | auto *I = cast<Instruction>(V); | |||
| 1452 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && | |||
| 1453 | Op3.match(I->getOperand(2)); | |||
| 1454 | } | |||
| 1455 | return false; | |||
| 1456 | } | |||
| 1457 | }; | |||
| 1458 | ||||
| 1459 | /// Matches SelectInst. | |||
| 1460 | template <typename Cond, typename LHS, typename RHS> | |||
| 1461 | inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select> | |||
| 1462 | m_Select(const Cond &C, const LHS &L, const RHS &R) { | |||
| 1463 | return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R); | |||
| 1464 | } | |||
| 1465 | ||||
| 1466 | /// This matches a select of two constants, e.g.: | |||
| 1467 | /// m_SelectCst<-1, 0>(m_Value(V)) | |||
| 1468 | template <int64_t L, int64_t R, typename Cond> | |||
| 1469 | inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>, | |||
| 1470 | Instruction::Select> | |||
| 1471 | m_SelectCst(const Cond &C) { | |||
| 1472 | return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>()); | |||
| 1473 | } | |||
| 1474 | ||||
| 1475 | /// Matches FreezeInst. | |||
| 1476 | template <typename OpTy> | |||
| 1477 | inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) { | |||
| 1478 | return OneOps_match<OpTy, Instruction::Freeze>(Op); | |||
| 1479 | } | |||
| 1480 | ||||
| 1481 | /// Matches InsertElementInst. | |||
| 1482 | template <typename Val_t, typename Elt_t, typename Idx_t> | |||
| 1483 | inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement> | |||
| 1484 | m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) { | |||
| 1485 | return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>( | |||
| 1486 | Val, Elt, Idx); | |||
| 1487 | } | |||
| 1488 | ||||
| 1489 | /// Matches ExtractElementInst. | |||
| 1490 | template <typename Val_t, typename Idx_t> | |||
| 1491 | inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement> | |||
| 1492 | m_ExtractElt(const Val_t &Val, const Idx_t &Idx) { | |||
| 1493 | return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx); | |||
| 1494 | } | |||
| 1495 | ||||
| 1496 | /// Matches shuffle. | |||
| 1497 | template <typename T0, typename T1, typename T2> struct Shuffle_match { | |||
| 1498 | T0 Op1; | |||
| 1499 | T1 Op2; | |||
| 1500 | T2 Mask; | |||
| 1501 | ||||
| 1502 | Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask) | |||
| 1503 | : Op1(Op1), Op2(Op2), Mask(Mask) {} | |||
| 1504 | ||||
| 1505 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1506 | if (auto *I = dyn_cast<ShuffleVectorInst>(V)) { | |||
| 1507 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && | |||
| 1508 | Mask.match(I->getShuffleMask()); | |||
| 1509 | } | |||
| 1510 | return false; | |||
| 1511 | } | |||
| 1512 | }; | |||
| 1513 | ||||
| 1514 | struct m_Mask { | |||
| 1515 | ArrayRef<int> &MaskRef; | |||
| 1516 | m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} | |||
| 1517 | bool match(ArrayRef<int> Mask) { | |||
| 1518 | MaskRef = Mask; | |||
| 1519 | return true; | |||
| 1520 | } | |||
| 1521 | }; | |||
| 1522 | ||||
| 1523 | struct m_ZeroMask { | |||
| 1524 | bool match(ArrayRef<int> Mask) { | |||
| 1525 | return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; }); | |||
| 1526 | } | |||
| 1527 | }; | |||
| 1528 | ||||
| 1529 | struct m_SpecificMask { | |||
| 1530 | ArrayRef<int> &MaskRef; | |||
| 1531 | m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} | |||
| 1532 | bool match(ArrayRef<int> Mask) { return MaskRef == Mask; } | |||
| 1533 | }; | |||
| 1534 | ||||
| 1535 | struct m_SplatOrUndefMask { | |||
| 1536 | int &SplatIndex; | |||
| 1537 | m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {} | |||
| 1538 | bool match(ArrayRef<int> Mask) { | |||
| 1539 | const auto *First = find_if(Mask, [](int Elem) { return Elem != -1; }); | |||
| 1540 | if (First == Mask.end()) | |||
| 1541 | return false; | |||
| 1542 | SplatIndex = *First; | |||
| 1543 | return all_of(Mask, | |||
| 1544 | [First](int Elem) { return Elem == *First || Elem == -1; }); | |||
| 1545 | } | |||
| 1546 | }; | |||
| 1547 | ||||
| 1548 | /// Matches ShuffleVectorInst independently of mask value. | |||
| 1549 | template <typename V1_t, typename V2_t> | |||
| 1550 | inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector> | |||
| 1551 | m_Shuffle(const V1_t &v1, const V2_t &v2) { | |||
| 1552 | return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2); | |||
| 1553 | } | |||
| 1554 | ||||
| 1555 | template <typename V1_t, typename V2_t, typename Mask_t> | |||
| 1556 | inline Shuffle_match<V1_t, V2_t, Mask_t> | |||
| 1557 | m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) { | |||
| 1558 | return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask); | |||
| 1559 | } | |||
| 1560 | ||||
| 1561 | /// Matches LoadInst. | |||
| 1562 | template <typename OpTy> | |||
| 1563 | inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) { | |||
| 1564 | return OneOps_match<OpTy, Instruction::Load>(Op); | |||
| 1565 | } | |||
| 1566 | ||||
| 1567 | /// Matches StoreInst. | |||
| 1568 | template <typename ValueOpTy, typename PointerOpTy> | |||
| 1569 | inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store> | |||
| 1570 | m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) { | |||
| 1571 | return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp, | |||
| 1572 | PointerOp); | |||
| 1573 | } | |||
| 1574 | ||||
| 1575 | //===----------------------------------------------------------------------===// | |||
| 1576 | // Matchers for CastInst classes | |||
| 1577 | // | |||
| 1578 | ||||
| 1579 | template <typename Op_t, unsigned Opcode> struct CastClass_match { | |||
| 1580 | Op_t Op; | |||
| 1581 | ||||
| 1582 | CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {} | |||
| 1583 | ||||
| 1584 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1585 | if (auto *O = dyn_cast<Operator>(V)) | |||
| 1586 | return O->getOpcode() == Opcode && Op.match(O->getOperand(0)); | |||
| 1587 | return false; | |||
| 1588 | } | |||
| 1589 | }; | |||
| 1590 | ||||
| 1591 | /// Matches BitCast. | |||
| 1592 | template <typename OpTy> | |||
| 1593 | inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) { | |||
| 1594 | return CastClass_match<OpTy, Instruction::BitCast>(Op); | |||
| 1595 | } | |||
| 1596 | ||||
| 1597 | /// Matches PtrToInt. | |||
| 1598 | template <typename OpTy> | |||
| 1599 | inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) { | |||
| 1600 | return CastClass_match<OpTy, Instruction::PtrToInt>(Op); | |||
| 1601 | } | |||
| 1602 | ||||
| 1603 | /// Matches IntToPtr. | |||
| 1604 | template <typename OpTy> | |||
| 1605 | inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) { | |||
| 1606 | return CastClass_match<OpTy, Instruction::IntToPtr>(Op); | |||
| 1607 | } | |||
| 1608 | ||||
| 1609 | /// Matches Trunc. | |||
| 1610 | template <typename OpTy> | |||
| 1611 | inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) { | |||
| 1612 | return CastClass_match<OpTy, Instruction::Trunc>(Op); | |||
| 1613 | } | |||
| 1614 | ||||
| 1615 | template <typename OpTy> | |||
| 1616 | inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy> | |||
| 1617 | m_TruncOrSelf(const OpTy &Op) { | |||
| 1618 | return m_CombineOr(m_Trunc(Op), Op); | |||
| 1619 | } | |||
| 1620 | ||||
| 1621 | /// Matches SExt. | |||
| 1622 | template <typename OpTy> | |||
| 1623 | inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) { | |||
| 1624 | return CastClass_match<OpTy, Instruction::SExt>(Op); | |||
| 1625 | } | |||
| 1626 | ||||
| 1627 | /// Matches ZExt. | |||
| 1628 | template <typename OpTy> | |||
| 1629 | inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { | |||
| 1630 | return CastClass_match<OpTy, Instruction::ZExt>(Op); | |||
| 1631 | } | |||
| 1632 | ||||
| 1633 | template <typename OpTy> | |||
| 1634 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy> | |||
| 1635 | m_ZExtOrSelf(const OpTy &Op) { | |||
| 1636 | return m_CombineOr(m_ZExt(Op), Op); | |||
| 1637 | } | |||
| 1638 | ||||
| 1639 | template <typename OpTy> | |||
| 1640 | inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy> | |||
| 1641 | m_SExtOrSelf(const OpTy &Op) { | |||
| 1642 | return m_CombineOr(m_SExt(Op), Op); | |||
| 1643 | } | |||
| 1644 | ||||
| 1645 | template <typename OpTy> | |||
| 1646 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, | |||
| 1647 | CastClass_match<OpTy, Instruction::SExt>> | |||
| 1648 | m_ZExtOrSExt(const OpTy &Op) { | |||
| 1649 | return m_CombineOr(m_ZExt(Op), m_SExt(Op)); | |||
| 1650 | } | |||
| 1651 | ||||
| 1652 | template <typename OpTy> | |||
| 1653 | inline match_combine_or< | |||
| 1654 | match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, | |||
| 1655 | CastClass_match<OpTy, Instruction::SExt>>, | |||
| 1656 | OpTy> | |||
| 1657 | m_ZExtOrSExtOrSelf(const OpTy &Op) { | |||
| 1658 | return m_CombineOr(m_ZExtOrSExt(Op), Op); | |||
| 1659 | } | |||
| 1660 | ||||
| 1661 | template <typename OpTy> | |||
| 1662 | inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) { | |||
| 1663 | return CastClass_match<OpTy, Instruction::UIToFP>(Op); | |||
| 1664 | } | |||
| 1665 | ||||
| 1666 | template <typename OpTy> | |||
| 1667 | inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) { | |||
| 1668 | return CastClass_match<OpTy, Instruction::SIToFP>(Op); | |||
| 1669 | } | |||
| 1670 | ||||
| 1671 | template <typename OpTy> | |||
| 1672 | inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) { | |||
| 1673 | return CastClass_match<OpTy, Instruction::FPToUI>(Op); | |||
| 1674 | } | |||
| 1675 | ||||
| 1676 | template <typename OpTy> | |||
| 1677 | inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) { | |||
| 1678 | return CastClass_match<OpTy, Instruction::FPToSI>(Op); | |||
| 1679 | } | |||
| 1680 | ||||
| 1681 | template <typename OpTy> | |||
| 1682 | inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) { | |||
| 1683 | return CastClass_match<OpTy, Instruction::FPTrunc>(Op); | |||
| 1684 | } | |||
| 1685 | ||||
| 1686 | template <typename OpTy> | |||
| 1687 | inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) { | |||
| 1688 | return CastClass_match<OpTy, Instruction::FPExt>(Op); | |||
| 1689 | } | |||
| 1690 | ||||
| 1691 | //===----------------------------------------------------------------------===// | |||
| 1692 | // Matchers for control flow. | |||
| 1693 | // | |||
| 1694 | ||||
| 1695 | struct br_match { | |||
| 1696 | BasicBlock *&Succ; | |||
| 1697 | ||||
| 1698 | br_match(BasicBlock *&Succ) : Succ(Succ) {} | |||
| 1699 | ||||
| 1700 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1701 | if (auto *BI = dyn_cast<BranchInst>(V)) | |||
| 1702 | if (BI->isUnconditional()) { | |||
| 1703 | Succ = BI->getSuccessor(0); | |||
| 1704 | return true; | |||
| 1705 | } | |||
| 1706 | return false; | |||
| 1707 | } | |||
| 1708 | }; | |||
| 1709 | ||||
| 1710 | inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } | |||
| 1711 | ||||
| 1712 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> | |||
| 1713 | struct brc_match { | |||
| 1714 | Cond_t Cond; | |||
| 1715 | TrueBlock_t T; | |||
| 1716 | FalseBlock_t F; | |||
| 1717 | ||||
| 1718 | brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f) | |||
| 1719 | : Cond(C), T(t), F(f) {} | |||
| 1720 | ||||
| 1721 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1722 | if (auto *BI = dyn_cast<BranchInst>(V)) | |||
| 1723 | if (BI->isConditional() && Cond.match(BI->getCondition())) | |||
| 1724 | return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1)); | |||
| 1725 | return false; | |||
| 1726 | } | |||
| 1727 | }; | |||
| 1728 | ||||
| 1729 | template <typename Cond_t> | |||
| 1730 | inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>> | |||
| 1731 | m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { | |||
| 1732 | return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>( | |||
| 1733 | C, m_BasicBlock(T), m_BasicBlock(F)); | |||
| 1734 | } | |||
| 1735 | ||||
| 1736 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> | |||
| 1737 | inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t> | |||
| 1738 | m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) { | |||
| 1739 | return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F); | |||
| 1740 | } | |||
| 1741 | ||||
| 1742 | //===----------------------------------------------------------------------===// | |||
| 1743 | // Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y). | |||
| 1744 | // | |||
| 1745 | ||||
| 1746 | template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t, | |||
| 1747 | bool Commutable = false> | |||
| 1748 | struct MaxMin_match { | |||
| 1749 | using PredType = Pred_t; | |||
| 1750 | LHS_t L; | |||
| 1751 | RHS_t R; | |||
| 1752 | ||||
| 1753 | // The evaluation order is always stable, regardless of Commutability. | |||
| 1754 | // The LHS is always matched first. | |||
| 1755 | MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | |||
| 1756 | ||||
| 1757 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1758 | if (auto *II = dyn_cast<IntrinsicInst>(V)) { | |||
| 1759 | Intrinsic::ID IID = II->getIntrinsicID(); | |||
| 1760 | if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) || | |||
| 1761 | (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) || | |||
| 1762 | (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) || | |||
| 1763 | (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) { | |||
| 1764 | Value *LHS = II->getOperand(0), *RHS = II->getOperand(1); | |||
| 1765 | return (L.match(LHS) && R.match(RHS)) || | |||
| 1766 | (Commutable && L.match(RHS) && R.match(LHS)); | |||
| 1767 | } | |||
| 1768 | } | |||
| 1769 | // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x". | |||
| 1770 | auto *SI = dyn_cast<SelectInst>(V); | |||
| 1771 | if (!SI) | |||
| 1772 | return false; | |||
| 1773 | auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition()); | |||
| 1774 | if (!Cmp) | |||
| 1775 | return false; | |||
| 1776 | // At this point we have a select conditioned on a comparison. Check that | |||
| 1777 | // it is the values returned by the select that are being compared. | |||
| 1778 | auto *TrueVal = SI->getTrueValue(); | |||
| 1779 | auto *FalseVal = SI->getFalseValue(); | |||
| 1780 | auto *LHS = Cmp->getOperand(0); | |||
| 1781 | auto *RHS = Cmp->getOperand(1); | |||
| 1782 | if ((TrueVal != LHS || FalseVal != RHS) && | |||
| 1783 | (TrueVal != RHS || FalseVal != LHS)) | |||
| 1784 | return false; | |||
| 1785 | typename CmpInst_t::Predicate Pred = | |||
| 1786 | LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate(); | |||
| 1787 | // Does "(x pred y) ? x : y" represent the desired max/min operation? | |||
| 1788 | if (!Pred_t::match(Pred)) | |||
| 1789 | return false; | |||
| 1790 | // It does! Bind the operands. | |||
| 1791 | return (L.match(LHS) && R.match(RHS)) || | |||
| 1792 | (Commutable && L.match(RHS) && R.match(LHS)); | |||
| 1793 | } | |||
| 1794 | }; | |||
| 1795 | ||||
| 1796 | /// Helper class for identifying signed max predicates. | |||
| 1797 | struct smax_pred_ty { | |||
| 1798 | static bool match(ICmpInst::Predicate Pred) { | |||
| 1799 | return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE; | |||
| 1800 | } | |||
| 1801 | }; | |||
| 1802 | ||||
| 1803 | /// Helper class for identifying signed min predicates. | |||
| 1804 | struct smin_pred_ty { | |||
| 1805 | static bool match(ICmpInst::Predicate Pred) { | |||
| 1806 | return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE; | |||
| 1807 | } | |||
| 1808 | }; | |||
| 1809 | ||||
| 1810 | /// Helper class for identifying unsigned max predicates. | |||
| 1811 | struct umax_pred_ty { | |||
| 1812 | static bool match(ICmpInst::Predicate Pred) { | |||
| 1813 | return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE; | |||
| 1814 | } | |||
| 1815 | }; | |||
| 1816 | ||||
| 1817 | /// Helper class for identifying unsigned min predicates. | |||
| 1818 | struct umin_pred_ty { | |||
| 1819 | static bool match(ICmpInst::Predicate Pred) { | |||
| 1820 | return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE; | |||
| 1821 | } | |||
| 1822 | }; | |||
| 1823 | ||||
| 1824 | /// Helper class for identifying ordered max predicates. | |||
| 1825 | struct ofmax_pred_ty { | |||
| 1826 | static bool match(FCmpInst::Predicate Pred) { | |||
| 1827 | return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE; | |||
| 1828 | } | |||
| 1829 | }; | |||
| 1830 | ||||
| 1831 | /// Helper class for identifying ordered min predicates. | |||
| 1832 | struct ofmin_pred_ty { | |||
| 1833 | static bool match(FCmpInst::Predicate Pred) { | |||
| 1834 | return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE; | |||
| 1835 | } | |||
| 1836 | }; | |||
| 1837 | ||||
| 1838 | /// Helper class for identifying unordered max predicates. | |||
| 1839 | struct ufmax_pred_ty { | |||
| 1840 | static bool match(FCmpInst::Predicate Pred) { | |||
| 1841 | return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE; | |||
| 1842 | } | |||
| 1843 | }; | |||
| 1844 | ||||
| 1845 | /// Helper class for identifying unordered min predicates. | |||
| 1846 | struct ufmin_pred_ty { | |||
| 1847 | static bool match(FCmpInst::Predicate Pred) { | |||
| 1848 | return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE; | |||
| 1849 | } | |||
| 1850 | }; | |||
| 1851 | ||||
| 1852 | template <typename LHS, typename RHS> | |||
| 1853 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L, | |||
| 1854 | const RHS &R) { | |||
| 1855 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R); | |||
| 1856 | } | |||
| 1857 | ||||
| 1858 | template <typename LHS, typename RHS> | |||
| 1859 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L, | |||
| 1860 | const RHS &R) { | |||
| 1861 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R); | |||
| 1862 | } | |||
| 1863 | ||||
| 1864 | template <typename LHS, typename RHS> | |||
| 1865 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L, | |||
| 1866 | const RHS &R) { | |||
| 1867 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R); | |||
| 1868 | } | |||
| 1869 | ||||
| 1870 | template <typename LHS, typename RHS> | |||
| 1871 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L, | |||
| 1872 | const RHS &R) { | |||
| 1873 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R); | |||
| 1874 | } | |||
| 1875 | ||||
| 1876 | template <typename LHS, typename RHS> | |||
| 1877 | inline match_combine_or< | |||
| 1878 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>, | |||
| 1879 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>, | |||
| 1880 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>, | |||
| 1881 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>> | |||
| 1882 | m_MaxOrMin(const LHS &L, const RHS &R) { | |||
| 1883 | return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)), | |||
| 1884 | m_CombineOr(m_UMax(L, R), m_UMin(L, R))); | |||
| 1885 | } | |||
| 1886 | ||||
| 1887 | /// Match an 'ordered' floating point maximum function. | |||
| 1888 | /// Floating point has one special value 'NaN'. Therefore, there is no total | |||
| 1889 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | |||
| 1890 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' | |||
| 1891 | /// semantics. In the presence of 'NaN' we have to preserve the original | |||
| 1892 | /// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate. | |||
| 1893 | /// | |||
| 1894 | /// max(L, R) iff L and R are not NaN | |||
| 1895 | /// m_OrdFMax(L, R) = R iff L or R are NaN | |||
| 1896 | template <typename LHS, typename RHS> | |||
| 1897 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L, | |||
| 1898 | const RHS &R) { | |||
| 1899 | return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R); | |||
| 1900 | } | |||
| 1901 | ||||
| 1902 | /// Match an 'ordered' floating point minimum function. | |||
| 1903 | /// Floating point has one special value 'NaN'. Therefore, there is no total | |||
| 1904 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | |||
| 1905 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' | |||
| 1906 | /// semantics. In the presence of 'NaN' we have to preserve the original | |||
| 1907 | /// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate. | |||
| 1908 | /// | |||
| 1909 | /// min(L, R) iff L and R are not NaN | |||
| 1910 | /// m_OrdFMin(L, R) = R iff L or R are NaN | |||
| 1911 | template <typename LHS, typename RHS> | |||
| 1912 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L, | |||
| 1913 | const RHS &R) { | |||
| 1914 | return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R); | |||
| 1915 | } | |||
| 1916 | ||||
| 1917 | /// Match an 'unordered' floating point maximum function. | |||
| 1918 | /// Floating point has one special value 'NaN'. Therefore, there is no total | |||
| 1919 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | |||
| 1920 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' | |||
| 1921 | /// semantics. In the presence of 'NaN' we have to preserve the original | |||
| 1922 | /// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate. | |||
| 1923 | /// | |||
| 1924 | /// max(L, R) iff L and R are not NaN | |||
| 1925 | /// m_UnordFMax(L, R) = L iff L or R are NaN | |||
| 1926 | template <typename LHS, typename RHS> | |||
| 1927 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty> | |||
| 1928 | m_UnordFMax(const LHS &L, const RHS &R) { | |||
| 1929 | return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R); | |||
| 1930 | } | |||
| 1931 | ||||
| 1932 | /// Match an 'unordered' floating point minimum function. | |||
| 1933 | /// Floating point has one special value 'NaN'. Therefore, there is no total | |||
| 1934 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | |||
| 1935 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' | |||
| 1936 | /// semantics. In the presence of 'NaN' we have to preserve the original | |||
| 1937 | /// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate. | |||
| 1938 | /// | |||
| 1939 | /// min(L, R) iff L and R are not NaN | |||
| 1940 | /// m_UnordFMin(L, R) = L iff L or R are NaN | |||
| 1941 | template <typename LHS, typename RHS> | |||
| 1942 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty> | |||
| 1943 | m_UnordFMin(const LHS &L, const RHS &R) { | |||
| 1944 | return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R); | |||
| 1945 | } | |||
| 1946 | ||||
| 1947 | //===----------------------------------------------------------------------===// | |||
| 1948 | // Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b | |||
| 1949 | // Note that S might be matched to other instructions than AddInst. | |||
| 1950 | // | |||
| 1951 | ||||
| 1952 | template <typename LHS_t, typename RHS_t, typename Sum_t> | |||
| 1953 | struct UAddWithOverflow_match { | |||
| 1954 | LHS_t L; | |||
| 1955 | RHS_t R; | |||
| 1956 | Sum_t S; | |||
| 1957 | ||||
| 1958 | UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S) | |||
| 1959 | : L(L), R(R), S(S) {} | |||
| 1960 | ||||
| 1961 | template <typename OpTy> bool match(OpTy *V) { | |||
| 1962 | Value *ICmpLHS, *ICmpRHS; | |||
| 1963 | ICmpInst::Predicate Pred; | |||
| 1964 | if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V)) | |||
| 1965 | return false; | |||
| 1966 | ||||
| 1967 | Value *AddLHS, *AddRHS; | |||
| 1968 | auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS)); | |||
| 1969 | ||||
| 1970 | // (a + b) u< a, (a + b) u< b | |||
| 1971 | if (Pred == ICmpInst::ICMP_ULT) | |||
| 1972 | if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS)) | |||
| 1973 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); | |||
| 1974 | ||||
| 1975 | // a >u (a + b), b >u (a + b) | |||
| 1976 | if (Pred == ICmpInst::ICMP_UGT) | |||
| 1977 | if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS)) | |||
| 1978 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); | |||
| 1979 | ||||
| 1980 | Value *Op1; | |||
| 1981 | auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes())); | |||
| 1982 | // (a ^ -1) <u b | |||
| 1983 | if (Pred == ICmpInst::ICMP_ULT) { | |||
| 1984 | if (XorExpr.match(ICmpLHS)) | |||
| 1985 | return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS); | |||
| 1986 | } | |||
| 1987 | // b > u (a ^ -1) | |||
| 1988 | if (Pred == ICmpInst::ICMP_UGT) { | |||
| 1989 | if (XorExpr.match(ICmpRHS)) | |||
| 1990 | return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS); | |||
| 1991 | } | |||
| 1992 | ||||
| 1993 | // Match special-case for increment-by-1. | |||
| 1994 | if (Pred == ICmpInst::ICMP_EQ) { | |||
| 1995 | // (a + 1) == 0 | |||
| 1996 | // (1 + a) == 0 | |||
| 1997 | if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) && | |||
| 1998 | (m_One().match(AddLHS) || m_One().match(AddRHS))) | |||
| 1999 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); | |||
| 2000 | // 0 == (a + 1) | |||
| 2001 | // 0 == (1 + a) | |||
| 2002 | if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) && | |||
| 2003 | (m_One().match(AddLHS) || m_One().match(AddRHS))) | |||
| 2004 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); | |||
| 2005 | } | |||
| 2006 | ||||
| 2007 | return false; | |||
| 2008 | } | |||
| 2009 | }; | |||
| 2010 | ||||
| 2011 | /// Match an icmp instruction checking for unsigned overflow on addition. | |||
| 2012 | /// | |||
| 2013 | /// S is matched to the addition whose result is being checked for overflow, and | |||
| 2014 | /// L and R are matched to the LHS and RHS of S. | |||
| 2015 | template <typename LHS_t, typename RHS_t, typename Sum_t> | |||
| 2016 | UAddWithOverflow_match<LHS_t, RHS_t, Sum_t> | |||
| 2017 | m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) { | |||
| 2018 | return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S); | |||
| 2019 | } | |||
| 2020 | ||||
| 2021 | template <typename Opnd_t> struct Argument_match { | |||
| 2022 | unsigned OpI; | |||
| 2023 | Opnd_t Val; | |||
| 2024 | ||||
| 2025 | Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} | |||
| 2026 | ||||
| 2027 | template <typename OpTy> bool match(OpTy *V) { | |||
| 2028 | // FIXME: Should likely be switched to use `CallBase`. | |||
| 2029 | if (const auto *CI = dyn_cast<CallInst>(V)) | |||
| 2030 | return Val.match(CI->getArgOperand(OpI)); | |||
| 2031 | return false; | |||
| 2032 | } | |||
| 2033 | }; | |||
| 2034 | ||||
| 2035 | /// Match an argument. | |||
| 2036 | template <unsigned OpI, typename Opnd_t> | |||
| 2037 | inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) { | |||
| 2038 | return Argument_match<Opnd_t>(OpI, Op); | |||
| 2039 | } | |||
| 2040 | ||||
| 2041 | /// Intrinsic matchers. | |||
| 2042 | struct IntrinsicID_match { | |||
| 2043 | unsigned ID; | |||
| 2044 | ||||
| 2045 | IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} | |||
| 2046 | ||||
| 2047 | template <typename OpTy> bool match(OpTy *V) { | |||
| 2048 | if (const auto *CI = dyn_cast<CallInst>(V)) | |||
| 2049 | if (const auto *F = CI->getCalledFunction()) | |||
| 2050 | return F->getIntrinsicID() == ID; | |||
| 2051 | return false; | |||
| 2052 | } | |||
| 2053 | }; | |||
| 2054 | ||||
| 2055 | /// Intrinsic matches are combinations of ID matchers, and argument | |||
| 2056 | /// matchers. Higher arity matcher are defined recursively in terms of and-ing | |||
| 2057 | /// them with lower arity matchers. Here's some convenient typedefs for up to | |||
| 2058 | /// several arguments, and more can be added as needed | |||
| 2059 | template <typename T0 = void, typename T1 = void, typename T2 = void, | |||
| 2060 | typename T3 = void, typename T4 = void, typename T5 = void, | |||
| 2061 | typename T6 = void, typename T7 = void, typename T8 = void, | |||
| 2062 | typename T9 = void, typename T10 = void> | |||
| 2063 | struct m_Intrinsic_Ty; | |||
| 2064 | template <typename T0> struct m_Intrinsic_Ty<T0> { | |||
| 2065 | using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>; | |||
| 2066 | }; | |||
| 2067 | template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { | |||
| 2068 | using Ty = | |||
| 2069 | match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>; | |||
| 2070 | }; | |||
| 2071 | template <typename T0, typename T1, typename T2> | |||
| 2072 | struct m_Intrinsic_Ty<T0, T1, T2> { | |||
| 2073 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, | |||
| 2074 | Argument_match<T2>>; | |||
| 2075 | }; | |||
| 2076 | template <typename T0, typename T1, typename T2, typename T3> | |||
| 2077 | struct m_Intrinsic_Ty<T0, T1, T2, T3> { | |||
| 2078 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, | |||
| 2079 | Argument_match<T3>>; | |||
| 2080 | }; | |||
| 2081 | ||||
| 2082 | template <typename T0, typename T1, typename T2, typename T3, typename T4> | |||
| 2083 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { | |||
| 2084 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty, | |||
| 2085 | Argument_match<T4>>; | |||
| 2086 | }; | |||
| 2087 | ||||
| 2088 | template <typename T0, typename T1, typename T2, typename T3, typename T4, | |||
| 2089 | typename T5> | |||
| 2090 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> { | |||
| 2091 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty, | |||
| 2092 | Argument_match<T5>>; | |||
| 2093 | }; | |||
| 2094 | ||||
| 2095 | /// Match intrinsic calls like this: | |||
| 2096 | /// m_Intrinsic<Intrinsic::fabs>(m_Value(X)) | |||
| 2097 | template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { | |||
| 2098 | return IntrinsicID_match(IntrID); | |||
| 2099 | } | |||
| 2100 | ||||
| 2101 | /// Matches MaskedLoad Intrinsic. | |||
| 2102 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> | |||
| 2103 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty | |||
| 2104 | m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, | |||
| 2105 | const Opnd3 &Op3) { | |||
| 2106 | return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3); | |||
| 2107 | } | |||
| 2108 | ||||
| 2109 | /// Matches MaskedGather Intrinsic. | |||
| 2110 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> | |||
| 2111 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty | |||
| 2112 | m_MaskedGather(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, | |||
| 2113 | const Opnd3 &Op3) { | |||
| 2114 | return m_Intrinsic<Intrinsic::masked_gather>(Op0, Op1, Op2, Op3); | |||
| 2115 | } | |||
| 2116 | ||||
| 2117 | template <Intrinsic::ID IntrID, typename T0> | |||
| 2118 | inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { | |||
| 2119 | return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); | |||
| 2120 | } | |||
| 2121 | ||||
| 2122 | template <Intrinsic::ID IntrID, typename T0, typename T1> | |||
| 2123 | inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0, | |||
| 2124 | const T1 &Op1) { | |||
| 2125 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1)); | |||
| 2126 | } | |||
| 2127 | ||||
| 2128 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2> | |||
| 2129 | inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty | |||
| 2130 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { | |||
| 2131 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2)); | |||
| 2132 | } | |||
| 2133 | ||||
| 2134 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | |||
| 2135 | typename T3> | |||
| 2136 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty | |||
| 2137 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { | |||
| 2138 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); | |||
| 2139 | } | |||
| 2140 | ||||
| 2141 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | |||
| 2142 | typename T3, typename T4> | |||
| 2143 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty | |||
| 2144 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, | |||
| 2145 | const T4 &Op4) { | |||
| 2146 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3), | |||
| 2147 | m_Argument<4>(Op4)); | |||
| 2148 | } | |||
| 2149 | ||||
| 2150 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | |||
| 2151 | typename T3, typename T4, typename T5> | |||
| 2152 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty | |||
| 2153 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, | |||
| 2154 | const T4 &Op4, const T5 &Op5) { | |||
| 2155 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4), | |||
| 2156 | m_Argument<5>(Op5)); | |||
| 2157 | } | |||
| 2158 | ||||
| 2159 | // Helper intrinsic matching specializations. | |||
| 2160 | template <typename Opnd0> | |||
| 2161 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) { | |||
| 2162 | return m_Intrinsic<Intrinsic::bitreverse>(Op0); | |||
| 2163 | } | |||
| 2164 | ||||
| 2165 | template <typename Opnd0> | |||
| 2166 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) { | |||
| 2167 | return m_Intrinsic<Intrinsic::bswap>(Op0); | |||
| 2168 | } | |||
| 2169 | ||||
| 2170 | template <typename Opnd0> | |||
| 2171 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) { | |||
| 2172 | return m_Intrinsic<Intrinsic::fabs>(Op0); | |||
| 2173 | } | |||
| 2174 | ||||
| 2175 | template <typename Opnd0> | |||
| 2176 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) { | |||
| 2177 | return m_Intrinsic<Intrinsic::canonicalize>(Op0); | |||
| 2178 | } | |||
| 2179 | ||||
| 2180 | template <typename Opnd0, typename Opnd1> | |||
| 2181 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0, | |||
| 2182 | const Opnd1 &Op1) { | |||
| 2183 | return m_Intrinsic<Intrinsic::minnum>(Op0, Op1); | |||
| 2184 | } | |||
| 2185 | ||||
| 2186 | template <typename Opnd0, typename Opnd1> | |||
| 2187 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0, | |||
| 2188 | const Opnd1 &Op1) { | |||
| 2189 | return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1); | |||
| 2190 | } | |||
| 2191 | ||||
| 2192 | template <typename Opnd0, typename Opnd1, typename Opnd2> | |||
| 2193 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty | |||
| 2194 | m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { | |||
| 2195 | return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2); | |||
| 2196 | } | |||
| 2197 | ||||
| 2198 | template <typename Opnd0, typename Opnd1, typename Opnd2> | |||
| 2199 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty | |||
| 2200 | m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { | |||
| 2201 | return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2); | |||
| 2202 | } | |||
| 2203 | ||||
| 2204 | template <typename Opnd0> | |||
| 2205 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_Sqrt(const Opnd0 &Op0) { | |||
| 2206 | return m_Intrinsic<Intrinsic::sqrt>(Op0); | |||
| 2207 | } | |||
| 2208 | ||||
| 2209 | template <typename Opnd0, typename Opnd1> | |||
| 2210 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_CopySign(const Opnd0 &Op0, | |||
| 2211 | const Opnd1 &Op1) { | |||
| 2212 | return m_Intrinsic<Intrinsic::copysign>(Op0, Op1); | |||
| 2213 | } | |||
| 2214 | ||||
| 2215 | template <typename Opnd0> | |||
| 2216 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_VecReverse(const Opnd0 &Op0) { | |||
| 2217 | return m_Intrinsic<Intrinsic::experimental_vector_reverse>(Op0); | |||
| 2218 | } | |||
| 2219 | ||||
| 2220 | //===----------------------------------------------------------------------===// | |||
| 2221 | // Matchers for two-operands operators with the operators in either order | |||
| 2222 | // | |||
| 2223 | ||||
| 2224 | /// Matches a BinaryOperator with LHS and RHS in either order. | |||
| 2225 | template <typename LHS, typename RHS> | |||
| 2226 | inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) { | |||
| 2227 | return AnyBinaryOp_match<LHS, RHS, true>(L, R); | |||
| 2228 | } | |||
| 2229 | ||||
| 2230 | /// Matches an ICmp with a predicate over LHS and RHS in either order. | |||
| 2231 | /// Swaps the predicate if operands are commuted. | |||
| 2232 | template <typename LHS, typename RHS> | |||
| 2233 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true> | |||
| 2234 | m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | |||
| 2235 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L, | |||
| 2236 | R); | |||
| 2237 | } | |||
| 2238 | ||||
| 2239 | /// Matches a specific opcode with LHS and RHS in either order. | |||
| 2240 | template <typename LHS, typename RHS> | |||
| 2241 | inline SpecificBinaryOp_match<LHS, RHS, true> | |||
| 2242 | m_c_BinOp(unsigned Opcode, const LHS &L, const RHS &R) { | |||
| 2243 | return SpecificBinaryOp_match<LHS, RHS, true>(Opcode, L, R); | |||
| 2244 | } | |||
| 2245 | ||||
| 2246 | /// Matches a Add with LHS and RHS in either order. | |||
| 2247 | template <typename LHS, typename RHS> | |||
| 2248 | inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L, | |||
| 2249 | const RHS &R) { | |||
| 2250 | return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R); | |||
| 2251 | } | |||
| 2252 | ||||
| 2253 | /// Matches a Mul with LHS and RHS in either order. | |||
| 2254 | template <typename LHS, typename RHS> | |||
| 2255 | inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L, | |||
| 2256 | const RHS &R) { | |||
| 2257 | return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R); | |||
| 2258 | } | |||
| 2259 | ||||
| 2260 | /// Matches an And with LHS and RHS in either order. | |||
| 2261 | template <typename LHS, typename RHS> | |||
| 2262 | inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L, | |||
| 2263 | const RHS &R) { | |||
| 2264 | return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R); | |||
| 2265 | } | |||
| 2266 | ||||
| 2267 | /// Matches an Or with LHS and RHS in either order. | |||
| 2268 | template <typename LHS, typename RHS> | |||
| 2269 | inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L, | |||
| 2270 | const RHS &R) { | |||
| 2271 | return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R); | |||
| 2272 | } | |||
| 2273 | ||||
| 2274 | /// Matches an Xor with LHS and RHS in either order. | |||
| 2275 | template <typename LHS, typename RHS> | |||
| 2276 | inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L, | |||
| 2277 | const RHS &R) { | |||
| 2278 | return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R); | |||
| 2279 | } | |||
| 2280 | ||||
| 2281 | /// Matches a 'Neg' as 'sub 0, V'. | |||
| 2282 | template <typename ValTy> | |||
| 2283 | inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub> | |||
| 2284 | m_Neg(const ValTy &V) { | |||
| 2285 | return m_Sub(m_ZeroInt(), V); | |||
| 2286 | } | |||
| 2287 | ||||
| 2288 | /// Matches a 'Neg' as 'sub nsw 0, V'. | |||
| 2289 | template <typename ValTy> | |||
| 2290 | inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, | |||
| 2291 | Instruction::Sub, | |||
| 2292 | OverflowingBinaryOperator::NoSignedWrap> | |||
| 2293 | m_NSWNeg(const ValTy &V) { | |||
| 2294 | return m_NSWSub(m_ZeroInt(), V); | |||
| 2295 | } | |||
| 2296 | ||||
| 2297 | /// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'. | |||
| 2298 | /// NOTE: we first match the 'Not' (by matching '-1'), | |||
| 2299 | /// and only then match the inner matcher! | |||
| 2300 | template <typename ValTy> | |||
| 2301 | inline BinaryOp_match<cst_pred_ty<is_all_ones>, ValTy, Instruction::Xor, true> | |||
| 2302 | m_Not(const ValTy &V) { | |||
| 2303 | return m_c_Xor(m_AllOnes(), V); | |||
| 2304 | } | |||
| 2305 | ||||
| 2306 | template <typename ValTy> struct NotForbidUndef_match { | |||
| 2307 | ValTy Val; | |||
| 2308 | NotForbidUndef_match(const ValTy &V) : Val(V) {} | |||
| 2309 | ||||
| 2310 | template <typename OpTy> bool match(OpTy *V) { | |||
| 2311 | // We do not use m_c_Xor because that could match an arbitrary APInt that is | |||
| 2312 | // not -1 as C and then fail to match the other operand if it is -1. | |||
| 2313 | // This code should still work even when both operands are constants. | |||
| 2314 | Value *X; | |||
| 2315 | const APInt *C; | |||
| 2316 | if (m_Xor(m_Value(X), m_APIntForbidUndef(C)).match(V) && C->isAllOnes()) | |||
| 2317 | return Val.match(X); | |||
| 2318 | if (m_Xor(m_APIntForbidUndef(C), m_Value(X)).match(V) && C->isAllOnes()) | |||
| 2319 | return Val.match(X); | |||
| 2320 | return false; | |||
| 2321 | } | |||
| 2322 | }; | |||
| 2323 | ||||
| 2324 | /// Matches a bitwise 'not' as 'xor V, -1' or 'xor -1, V'. For vectors, the | |||
| 2325 | /// constant value must be composed of only -1 scalar elements. | |||
| 2326 | template <typename ValTy> | |||
| 2327 | inline NotForbidUndef_match<ValTy> m_NotForbidUndef(const ValTy &V) { | |||
| 2328 | return NotForbidUndef_match<ValTy>(V); | |||
| 2329 | } | |||
| 2330 | ||||
| 2331 | /// Matches an SMin with LHS and RHS in either order. | |||
| 2332 | template <typename LHS, typename RHS> | |||
| 2333 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true> | |||
| 2334 | m_c_SMin(const LHS &L, const RHS &R) { | |||
| 2335 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R); | |||
| 2336 | } | |||
| 2337 | /// Matches an SMax with LHS and RHS in either order. | |||
| 2338 | template <typename LHS, typename RHS> | |||
| 2339 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true> | |||
| 2340 | m_c_SMax(const LHS &L, const RHS &R) { | |||
| 2341 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R); | |||
| 2342 | } | |||
| 2343 | /// Matches a UMin with LHS and RHS in either order. | |||
| 2344 | template <typename LHS, typename RHS> | |||
| 2345 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true> | |||
| 2346 | m_c_UMin(const LHS &L, const RHS &R) { | |||
| 2347 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R); | |||
| 2348 | } | |||
| 2349 | /// Matches a UMax with LHS and RHS in either order. | |||
| 2350 | template <typename LHS, typename RHS> | |||
| 2351 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true> | |||
| 2352 | m_c_UMax(const LHS &L, const RHS &R) { | |||
| 2353 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R); | |||
| 2354 | } | |||
| 2355 | ||||
| 2356 | template <typename LHS, typename RHS> | |||
| 2357 | inline match_combine_or< | |||
| 2358 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>, | |||
| 2359 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>, | |||
| 2360 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>, | |||
| 2361 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>> | |||
| 2362 | m_c_MaxOrMin(const LHS &L, const RHS &R) { | |||
| 2363 | return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)), | |||
| 2364 | m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R))); | |||
| 2365 | } | |||
| 2366 | ||||
| 2367 | template <Intrinsic::ID IntrID, typename T0, typename T1> | |||
| 2368 | inline match_combine_or<typename m_Intrinsic_Ty<T0, T1>::Ty, | |||
| 2369 | typename m_Intrinsic_Ty<T1, T0>::Ty> | |||
| 2370 | m_c_Intrinsic(const T0 &Op0, const T1 &Op1) { | |||
| 2371 | return m_CombineOr(m_Intrinsic<IntrID>(Op0, Op1), | |||
| 2372 | m_Intrinsic<IntrID>(Op1, Op0)); | |||
| 2373 | } | |||
| 2374 | ||||
| 2375 | /// Matches FAdd with LHS and RHS in either order. | |||
| 2376 | template <typename LHS, typename RHS> | |||
| 2377 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true> | |||
| 2378 | m_c_FAdd(const LHS &L, const RHS &R) { | |||
| 2379 | return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R); | |||
| 2380 | } | |||
| 2381 | ||||
| 2382 | /// Matches FMul with LHS and RHS in either order. | |||
| 2383 | template <typename LHS, typename RHS> | |||
| 2384 | inline BinaryOp_match<LHS, RHS, Instruction::FMul, true> | |||
| 2385 | m_c_FMul(const LHS &L, const RHS &R) { | |||
| 2386 | return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R); | |||
| 2387 | } | |||
| 2388 | ||||
| 2389 | template <typename Opnd_t> struct Signum_match { | |||
| 2390 | Opnd_t Val; | |||
| 2391 | Signum_match(const Opnd_t &V) : Val(V) {} | |||
| 2392 | ||||
| 2393 | template <typename OpTy> bool match(OpTy *V) { | |||
| 2394 | unsigned TypeSize = V->getType()->getScalarSizeInBits(); | |||
| 2395 | if (TypeSize == 0) | |||
| 2396 | return false; | |||
| 2397 | ||||
| 2398 | unsigned ShiftWidth = TypeSize - 1; | |||
| 2399 | Value *OpL = nullptr, *OpR = nullptr; | |||
| 2400 | ||||
| 2401 | // This is the representation of signum we match: | |||
| 2402 | // | |||
| 2403 | // signum(x) == (x >> 63) | (-x >>u 63) | |||
| 2404 | // | |||
| 2405 | // An i1 value is its own signum, so it's correct to match | |||
| 2406 | // | |||
| 2407 | // signum(x) == (x >> 0) | (-x >>u 0) | |||
| 2408 | // | |||
| 2409 | // for i1 values. | |||
| 2410 | ||||
| 2411 | auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth)); | |||
| 2412 | auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth)); | |||
| 2413 | auto Signum = m_Or(LHS, RHS); | |||
| 2414 | ||||
| 2415 | return Signum.match(V) && OpL == OpR && Val.match(OpL); | |||
| 2416 | } | |||
| 2417 | }; | |||
| 2418 | ||||
| 2419 | /// Matches a signum pattern. | |||
| 2420 | /// | |||
| 2421 | /// signum(x) = | |||
| 2422 | /// x > 0 -> 1 | |||
| 2423 | /// x == 0 -> 0 | |||
| 2424 | /// x < 0 -> -1 | |||
| 2425 | template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { | |||
| 2426 | return Signum_match<Val_t>(V); | |||
| 2427 | } | |||
| 2428 | ||||
| 2429 | template <int Ind, typename Opnd_t> struct ExtractValue_match { | |||
| 2430 | Opnd_t Val; | |||
| 2431 | ExtractValue_match(const Opnd_t &V) : Val(V) {} | |||
| 2432 | ||||
| 2433 | template <typename OpTy> bool match(OpTy *V) { | |||
| 2434 | if (auto *I = dyn_cast<ExtractValueInst>(V)) { | |||
| 2435 | // If Ind is -1, don't inspect indices | |||
| 2436 | if (Ind != -1 && | |||
| 2437 | !(I->getNumIndices() == 1 && I->getIndices()[0] == (unsigned)Ind)) | |||
| 2438 | return false; | |||
| 2439 | return Val.match(I->getAggregateOperand()); | |||
| 2440 | } | |||
| 2441 | return false; | |||
| 2442 | } | |||
| 2443 | }; | |||
| 2444 | ||||
| 2445 | /// Match a single index ExtractValue instruction. | |||
| 2446 | /// For example m_ExtractValue<1>(...) | |||
| 2447 | template <int Ind, typename Val_t> | |||
| 2448 | inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) { | |||
| 2449 | return ExtractValue_match<Ind, Val_t>(V); | |||
| 2450 | } | |||
| 2451 | ||||
| 2452 | /// Match an ExtractValue instruction with any index. | |||
| 2453 | /// For example m_ExtractValue(...) | |||
| 2454 | template <typename Val_t> | |||
| 2455 | inline ExtractValue_match<-1, Val_t> m_ExtractValue(const Val_t &V) { | |||
| 2456 | return ExtractValue_match<-1, Val_t>(V); | |||
| 2457 | } | |||
| 2458 | ||||
| 2459 | /// Matcher for a single index InsertValue instruction. | |||
| 2460 | template <int Ind, typename T0, typename T1> struct InsertValue_match { | |||
| 2461 | T0 Op0; | |||
| 2462 | T1 Op1; | |||
| 2463 | ||||
| 2464 | InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {} | |||
| 2465 | ||||
| 2466 | template <typename OpTy> bool match(OpTy *V) { | |||
| 2467 | if (auto *I = dyn_cast<InsertValueInst>(V)) { | |||
| 2468 | return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) && | |||
| 2469 | I->getNumIndices() == 1 && Ind == I->getIndices()[0]; | |||
| 2470 | } | |||
| 2471 | return false; | |||
| 2472 | } | |||
| 2473 | }; | |||
| 2474 | ||||
| 2475 | /// Matches a single index InsertValue instruction. | |||
| 2476 | template <int Ind, typename Val_t, typename Elt_t> | |||
| 2477 | inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val, | |||
| 2478 | const Elt_t &Elt) { | |||
| 2479 | return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt); | |||
| 2480 | } | |||
| 2481 | ||||
| 2482 | /// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or | |||
| 2483 | /// the constant expression | |||
| 2484 | /// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>` | |||
| 2485 | /// under the right conditions determined by DataLayout. | |||
| 2486 | struct VScaleVal_match { | |||
| 2487 | template <typename ITy> bool match(ITy *V) { | |||
| 2488 | if (m_Intrinsic<Intrinsic::vscale>().match(V)) | |||
| 2489 | return true; | |||
| 2490 | ||||
| 2491 | Value *Ptr; | |||
| 2492 | if (m_PtrToInt(m_Value(Ptr)).match(V)) { | |||
| 2493 | if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) { | |||
| 2494 | auto *DerefTy = | |||
| 2495 | dyn_cast<ScalableVectorType>(GEP->getSourceElementType()); | |||
| 2496 | if (GEP->getNumIndices() == 1 && DerefTy && | |||
| 2497 | DerefTy->getElementType()->isIntegerTy(8) && | |||
| 2498 | m_Zero().match(GEP->getPointerOperand()) && | |||
| 2499 | m_SpecificInt(1).match(GEP->idx_begin()->get())) | |||
| 2500 | return true; | |||
| 2501 | } | |||
| 2502 | } | |||
| 2503 | ||||
| 2504 | return false; | |||
| 2505 | } | |||
| 2506 | }; | |||
| 2507 | ||||
| 2508 | inline VScaleVal_match m_VScale() { | |||
| 2509 | return VScaleVal_match(); | |||
| 2510 | } | |||
| 2511 | ||||
| 2512 | template <typename LHS, typename RHS, unsigned Opcode, bool Commutable = false> | |||
| 2513 | struct LogicalOp_match { | |||
| 2514 | LHS L; | |||
| 2515 | RHS R; | |||
| 2516 | ||||
| 2517 | LogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {} | |||
| 2518 | ||||
| 2519 | template <typename T> bool match(T *V) { | |||
| 2520 | auto *I = dyn_cast<Instruction>(V); | |||
| 2521 | if (!I || !I->getType()->isIntOrIntVectorTy(1)) | |||
| 2522 | return false; | |||
| 2523 | ||||
| 2524 | if (I->getOpcode() == Opcode) { | |||
| 2525 | auto *Op0 = I->getOperand(0); | |||
| 2526 | auto *Op1 = I->getOperand(1); | |||
| 2527 | return (L.match(Op0) && R.match(Op1)) || | |||
| 2528 | (Commutable && L.match(Op1) && R.match(Op0)); | |||
| 2529 | } | |||
| 2530 | ||||
| 2531 | if (auto *Select = dyn_cast<SelectInst>(I)) { | |||
| 2532 | auto *Cond = Select->getCondition(); | |||
| 2533 | auto *TVal = Select->getTrueValue(); | |||
| 2534 | auto *FVal = Select->getFalseValue(); | |||
| 2535 | ||||
| 2536 | // Don't match a scalar select of bool vectors. | |||
| 2537 | // Transforms expect a single type for operands if this matches. | |||
| 2538 | if (Cond->getType() != Select->getType()) | |||
| 2539 | return false; | |||
| 2540 | ||||
| 2541 | if (Opcode == Instruction::And) { | |||
| 2542 | auto *C = dyn_cast<Constant>(FVal); | |||
| 2543 | if (C && C->isNullValue()) | |||
| 2544 | return (L.match(Cond) && R.match(TVal)) || | |||
| 2545 | (Commutable && L.match(TVal) && R.match(Cond)); | |||
| 2546 | } else { | |||
| 2547 | assert(Opcode == Instruction::Or)(static_cast <bool> (Opcode == Instruction::Or) ? void ( 0) : __assert_fail ("Opcode == Instruction::Or", "llvm/include/llvm/IR/PatternMatch.h" , 2547, __extension__ __PRETTY_FUNCTION__)); | |||
| 2548 | auto *C = dyn_cast<Constant>(TVal); | |||
| 2549 | if (C && C->isOneValue()) | |||
| 2550 | return (L.match(Cond) && R.match(FVal)) || | |||
| 2551 | (Commutable && L.match(FVal) && R.match(Cond)); | |||
| 2552 | } | |||
| 2553 | } | |||
| 2554 | ||||
| 2555 | return false; | |||
| 2556 | } | |||
| 2557 | }; | |||
| 2558 | ||||
| 2559 | /// Matches L && R either in the form of L & R or L ? R : false. | |||
| 2560 | /// Note that the latter form is poison-blocking. | |||
| 2561 | template <typename LHS, typename RHS> | |||
| 2562 | inline LogicalOp_match<LHS, RHS, Instruction::And> m_LogicalAnd(const LHS &L, | |||
| 2563 | const RHS &R) { | |||
| 2564 | return LogicalOp_match<LHS, RHS, Instruction::And>(L, R); | |||
| 2565 | } | |||
| 2566 | ||||
| 2567 | /// Matches L && R where L and R are arbitrary values. | |||
| 2568 | inline auto m_LogicalAnd() { return m_LogicalAnd(m_Value(), m_Value()); } | |||
| 2569 | ||||
| 2570 | /// Matches L && R with LHS and RHS in either order. | |||
| 2571 | template <typename LHS, typename RHS> | |||
| 2572 | inline LogicalOp_match<LHS, RHS, Instruction::And, true> | |||
| 2573 | m_c_LogicalAnd(const LHS &L, const RHS &R) { | |||
| 2574 | return LogicalOp_match<LHS, RHS, Instruction::And, true>(L, R); | |||
| 2575 | } | |||
| 2576 | ||||
| 2577 | /// Matches L || R either in the form of L | R or L ? true : R. | |||
| 2578 | /// Note that the latter form is poison-blocking. | |||
| 2579 | template <typename LHS, typename RHS> | |||
| 2580 | inline LogicalOp_match<LHS, RHS, Instruction::Or> m_LogicalOr(const LHS &L, | |||
| 2581 | const RHS &R) { | |||
| 2582 | return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R); | |||
| 2583 | } | |||
| 2584 | ||||
| 2585 | /// Matches L || R where L and R are arbitrary values. | |||
| 2586 | inline auto m_LogicalOr() { return m_LogicalOr(m_Value(), m_Value()); } | |||
| 2587 | ||||
| 2588 | /// Matches L || R with LHS and RHS in either order. | |||
| 2589 | template <typename LHS, typename RHS> | |||
| 2590 | inline LogicalOp_match<LHS, RHS, Instruction::Or, true> | |||
| 2591 | m_c_LogicalOr(const LHS &L, const RHS &R) { | |||
| 2592 | return LogicalOp_match<LHS, RHS, Instruction::Or, true>(L, R); | |||
| 2593 | } | |||
| 2594 | ||||
| 2595 | /// Matches either L && R or L || R, | |||
| 2596 | /// either one being in the either binary or logical form. | |||
| 2597 | /// Note that the latter form is poison-blocking. | |||
| 2598 | template <typename LHS, typename RHS, bool Commutable = false> | |||
| 2599 | inline auto m_LogicalOp(const LHS &L, const RHS &R) { | |||
| 2600 | return m_CombineOr( | |||
| 2601 | LogicalOp_match<LHS, RHS, Instruction::And, Commutable>(L, R), | |||
| 2602 | LogicalOp_match<LHS, RHS, Instruction::Or, Commutable>(L, R)); | |||
| 2603 | } | |||
| 2604 | ||||
| 2605 | /// Matches either L && R or L || R where L and R are arbitrary values. | |||
| 2606 | inline auto m_LogicalOp() { return m_LogicalOp(m_Value(), m_Value()); } | |||
| 2607 | ||||
| 2608 | /// Matches either L && R or L || R with LHS and RHS in either order. | |||
| 2609 | template <typename LHS, typename RHS> | |||
| 2610 | inline auto m_c_LogicalOp(const LHS &L, const RHS &R) { | |||
| 2611 | return m_LogicalOp<LHS, RHS, /*Commutable=*/true>(L, R); | |||
| 2612 | } | |||
| 2613 | ||||
| 2614 | } // end namespace PatternMatch | |||
| 2615 | } // end namespace llvm | |||
| 2616 | ||||
| 2617 | #endif // LLVM_IR_PATTERNMATCH_H |