| File: | llvm/lib/Analysis/VectorUtils.cpp |
| Warning: | line 1180, column 11 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===----------- VectorUtils.cpp - Vectorizer utility functions -----------===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // This file defines vectorizer utilities. | ||||||||
| 10 | // | ||||||||
| 11 | //===----------------------------------------------------------------------===// | ||||||||
| 12 | |||||||||
| 13 | #include "llvm/Analysis/VectorUtils.h" | ||||||||
| 14 | #include "llvm/ADT/EquivalenceClasses.h" | ||||||||
| 15 | #include "llvm/Analysis/DemandedBits.h" | ||||||||
| 16 | #include "llvm/Analysis/LoopInfo.h" | ||||||||
| 17 | #include "llvm/Analysis/LoopIterator.h" | ||||||||
| 18 | #include "llvm/Analysis/ScalarEvolution.h" | ||||||||
| 19 | #include "llvm/Analysis/ScalarEvolutionExpressions.h" | ||||||||
| 20 | #include "llvm/Analysis/TargetTransformInfo.h" | ||||||||
| 21 | #include "llvm/Analysis/ValueTracking.h" | ||||||||
| 22 | #include "llvm/IR/Constants.h" | ||||||||
| 23 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||||||
| 24 | #include "llvm/IR/IRBuilder.h" | ||||||||
| 25 | #include "llvm/IR/PatternMatch.h" | ||||||||
| 26 | #include "llvm/IR/Value.h" | ||||||||
| 27 | #include "llvm/Support/CommandLine.h" | ||||||||
| 28 | |||||||||
| 29 | #define DEBUG_TYPE"vectorutils" "vectorutils" | ||||||||
| 30 | |||||||||
| 31 | using namespace llvm; | ||||||||
| 32 | using namespace llvm::PatternMatch; | ||||||||
| 33 | |||||||||
| 34 | /// Maximum factor for an interleaved memory access. | ||||||||
| 35 | static cl::opt<unsigned> MaxInterleaveGroupFactor( | ||||||||
| 36 | "max-interleave-group-factor", cl::Hidden, | ||||||||
| 37 | cl::desc("Maximum factor for an interleaved access group (default = 8)"), | ||||||||
| 38 | cl::init(8)); | ||||||||
| 39 | |||||||||
| 40 | /// Return true if all of the intrinsic's arguments and return type are scalars | ||||||||
| 41 | /// for the scalar form of the intrinsic, and vectors for the vector form of the | ||||||||
| 42 | /// intrinsic (except operands that are marked as always being scalar by | ||||||||
| 43 | /// hasVectorInstrinsicScalarOpd). | ||||||||
| 44 | bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { | ||||||||
| 45 | switch (ID) { | ||||||||
| 46 | case Intrinsic::abs: // Begin integer bit-manipulation. | ||||||||
| 47 | case Intrinsic::bswap: | ||||||||
| 48 | case Intrinsic::bitreverse: | ||||||||
| 49 | case Intrinsic::ctpop: | ||||||||
| 50 | case Intrinsic::ctlz: | ||||||||
| 51 | case Intrinsic::cttz: | ||||||||
| 52 | case Intrinsic::fshl: | ||||||||
| 53 | case Intrinsic::fshr: | ||||||||
| 54 | case Intrinsic::smax: | ||||||||
| 55 | case Intrinsic::smin: | ||||||||
| 56 | case Intrinsic::umax: | ||||||||
| 57 | case Intrinsic::umin: | ||||||||
| 58 | case Intrinsic::sadd_sat: | ||||||||
| 59 | case Intrinsic::ssub_sat: | ||||||||
| 60 | case Intrinsic::uadd_sat: | ||||||||
| 61 | case Intrinsic::usub_sat: | ||||||||
| 62 | case Intrinsic::smul_fix: | ||||||||
| 63 | case Intrinsic::smul_fix_sat: | ||||||||
| 64 | case Intrinsic::umul_fix: | ||||||||
| 65 | case Intrinsic::umul_fix_sat: | ||||||||
| 66 | case Intrinsic::sqrt: // Begin floating-point. | ||||||||
| 67 | case Intrinsic::sin: | ||||||||
| 68 | case Intrinsic::cos: | ||||||||
| 69 | case Intrinsic::exp: | ||||||||
| 70 | case Intrinsic::exp2: | ||||||||
| 71 | case Intrinsic::log: | ||||||||
| 72 | case Intrinsic::log10: | ||||||||
| 73 | case Intrinsic::log2: | ||||||||
| 74 | case Intrinsic::fabs: | ||||||||
| 75 | case Intrinsic::minnum: | ||||||||
| 76 | case Intrinsic::maxnum: | ||||||||
| 77 | case Intrinsic::minimum: | ||||||||
| 78 | case Intrinsic::maximum: | ||||||||
| 79 | case Intrinsic::copysign: | ||||||||
| 80 | case Intrinsic::floor: | ||||||||
| 81 | case Intrinsic::ceil: | ||||||||
| 82 | case Intrinsic::trunc: | ||||||||
| 83 | case Intrinsic::rint: | ||||||||
| 84 | case Intrinsic::nearbyint: | ||||||||
| 85 | case Intrinsic::round: | ||||||||
| 86 | case Intrinsic::roundeven: | ||||||||
| 87 | case Intrinsic::pow: | ||||||||
| 88 | case Intrinsic::fma: | ||||||||
| 89 | case Intrinsic::fmuladd: | ||||||||
| 90 | case Intrinsic::powi: | ||||||||
| 91 | case Intrinsic::canonicalize: | ||||||||
| 92 | return true; | ||||||||
| 93 | default: | ||||||||
| 94 | return false; | ||||||||
| 95 | } | ||||||||
| 96 | } | ||||||||
| 97 | |||||||||
| 98 | /// Identifies if the vector form of the intrinsic has a scalar operand. | ||||||||
| 99 | bool llvm::hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, | ||||||||
| 100 | unsigned ScalarOpdIdx) { | ||||||||
| 101 | switch (ID) { | ||||||||
| 102 | case Intrinsic::abs: | ||||||||
| 103 | case Intrinsic::ctlz: | ||||||||
| 104 | case Intrinsic::cttz: | ||||||||
| 105 | case Intrinsic::powi: | ||||||||
| 106 | return (ScalarOpdIdx == 1); | ||||||||
| 107 | case Intrinsic::smul_fix: | ||||||||
| 108 | case Intrinsic::smul_fix_sat: | ||||||||
| 109 | case Intrinsic::umul_fix: | ||||||||
| 110 | case Intrinsic::umul_fix_sat: | ||||||||
| 111 | return (ScalarOpdIdx == 2); | ||||||||
| 112 | default: | ||||||||
| 113 | return false; | ||||||||
| 114 | } | ||||||||
| 115 | } | ||||||||
| 116 | |||||||||
| 117 | bool llvm::hasVectorInstrinsicOverloadedScalarOpd(Intrinsic::ID ID, | ||||||||
| 118 | unsigned ScalarOpdIdx) { | ||||||||
| 119 | switch (ID) { | ||||||||
| 120 | case Intrinsic::powi: | ||||||||
| 121 | return (ScalarOpdIdx == 1); | ||||||||
| 122 | default: | ||||||||
| 123 | return false; | ||||||||
| 124 | } | ||||||||
| 125 | } | ||||||||
| 126 | |||||||||
| 127 | /// Returns intrinsic ID for call. | ||||||||
| 128 | /// For the input call instruction it finds mapping intrinsic and returns | ||||||||
| 129 | /// its ID, in case it does not found it return not_intrinsic. | ||||||||
| 130 | Intrinsic::ID llvm::getVectorIntrinsicIDForCall(const CallInst *CI, | ||||||||
| 131 | const TargetLibraryInfo *TLI) { | ||||||||
| 132 | Intrinsic::ID ID = getIntrinsicForCallSite(*CI, TLI); | ||||||||
| 133 | if (ID == Intrinsic::not_intrinsic) | ||||||||
| 134 | return Intrinsic::not_intrinsic; | ||||||||
| 135 | |||||||||
| 136 | if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start || | ||||||||
| 137 | ID == Intrinsic::lifetime_end || ID == Intrinsic::assume || | ||||||||
| 138 | ID == Intrinsic::experimental_noalias_scope_decl || | ||||||||
| 139 | ID == Intrinsic::sideeffect || ID == Intrinsic::pseudoprobe) | ||||||||
| 140 | return ID; | ||||||||
| 141 | return Intrinsic::not_intrinsic; | ||||||||
| 142 | } | ||||||||
| 143 | |||||||||
| 144 | /// Find the operand of the GEP that should be checked for consecutive | ||||||||
| 145 | /// stores. This ignores trailing indices that have no effect on the final | ||||||||
| 146 | /// pointer. | ||||||||
| 147 | unsigned llvm::getGEPInductionOperand(const GetElementPtrInst *Gep) { | ||||||||
| 148 | const DataLayout &DL = Gep->getModule()->getDataLayout(); | ||||||||
| 149 | unsigned LastOperand = Gep->getNumOperands() - 1; | ||||||||
| 150 | TypeSize GEPAllocSize = DL.getTypeAllocSize(Gep->getResultElementType()); | ||||||||
| 151 | |||||||||
| 152 | // Walk backwards and try to peel off zeros. | ||||||||
| 153 | while (LastOperand > 1 && match(Gep->getOperand(LastOperand), m_Zero())) { | ||||||||
| 154 | // Find the type we're currently indexing into. | ||||||||
| 155 | gep_type_iterator GEPTI = gep_type_begin(Gep); | ||||||||
| 156 | std::advance(GEPTI, LastOperand - 2); | ||||||||
| 157 | |||||||||
| 158 | // If it's a type with the same allocation size as the result of the GEP we | ||||||||
| 159 | // can peel off the zero index. | ||||||||
| 160 | if (DL.getTypeAllocSize(GEPTI.getIndexedType()) != GEPAllocSize) | ||||||||
| 161 | break; | ||||||||
| 162 | --LastOperand; | ||||||||
| 163 | } | ||||||||
| 164 | |||||||||
| 165 | return LastOperand; | ||||||||
| 166 | } | ||||||||
| 167 | |||||||||
| 168 | /// If the argument is a GEP, then returns the operand identified by | ||||||||
| 169 | /// getGEPInductionOperand. However, if there is some other non-loop-invariant | ||||||||
| 170 | /// operand, it returns that instead. | ||||||||
| 171 | Value *llvm::stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp) { | ||||||||
| 172 | GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr); | ||||||||
| 173 | if (!GEP) | ||||||||
| 174 | return Ptr; | ||||||||
| 175 | |||||||||
| 176 | unsigned InductionOperand = getGEPInductionOperand(GEP); | ||||||||
| 177 | |||||||||
| 178 | // Check that all of the gep indices are uniform except for our induction | ||||||||
| 179 | // operand. | ||||||||
| 180 | for (unsigned i = 0, e = GEP->getNumOperands(); i != e; ++i) | ||||||||
| 181 | if (i != InductionOperand && | ||||||||
| 182 | !SE->isLoopInvariant(SE->getSCEV(GEP->getOperand(i)), Lp)) | ||||||||
| 183 | return Ptr; | ||||||||
| 184 | return GEP->getOperand(InductionOperand); | ||||||||
| 185 | } | ||||||||
| 186 | |||||||||
| 187 | /// If a value has only one user that is a CastInst, return it. | ||||||||
| 188 | Value *llvm::getUniqueCastUse(Value *Ptr, Loop *Lp, Type *Ty) { | ||||||||
| 189 | Value *UniqueCast = nullptr; | ||||||||
| 190 | for (User *U : Ptr->users()) { | ||||||||
| 191 | CastInst *CI = dyn_cast<CastInst>(U); | ||||||||
| 192 | if (CI && CI->getType() == Ty) { | ||||||||
| 193 | if (!UniqueCast) | ||||||||
| 194 | UniqueCast = CI; | ||||||||
| 195 | else | ||||||||
| 196 | return nullptr; | ||||||||
| 197 | } | ||||||||
| 198 | } | ||||||||
| 199 | return UniqueCast; | ||||||||
| 200 | } | ||||||||
| 201 | |||||||||
| 202 | /// Get the stride of a pointer access in a loop. Looks for symbolic | ||||||||
| 203 | /// strides "a[i*stride]". Returns the symbolic stride, or null otherwise. | ||||||||
| 204 | Value *llvm::getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp) { | ||||||||
| 205 | auto *PtrTy = dyn_cast<PointerType>(Ptr->getType()); | ||||||||
| 206 | if (!PtrTy || PtrTy->isAggregateType()) | ||||||||
| 207 | return nullptr; | ||||||||
| 208 | |||||||||
| 209 | // Try to remove a gep instruction to make the pointer (actually index at this | ||||||||
| 210 | // point) easier analyzable. If OrigPtr is equal to Ptr we are analyzing the | ||||||||
| 211 | // pointer, otherwise, we are analyzing the index. | ||||||||
| 212 | Value *OrigPtr = Ptr; | ||||||||
| 213 | |||||||||
| 214 | // The size of the pointer access. | ||||||||
| 215 | int64_t PtrAccessSize = 1; | ||||||||
| 216 | |||||||||
| 217 | Ptr = stripGetElementPtr(Ptr, SE, Lp); | ||||||||
| 218 | const SCEV *V = SE->getSCEV(Ptr); | ||||||||
| 219 | |||||||||
| 220 | if (Ptr != OrigPtr) | ||||||||
| 221 | // Strip off casts. | ||||||||
| 222 | while (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(V)) | ||||||||
| 223 | V = C->getOperand(); | ||||||||
| 224 | |||||||||
| 225 | const SCEVAddRecExpr *S = dyn_cast<SCEVAddRecExpr>(V); | ||||||||
| 226 | if (!S) | ||||||||
| 227 | return nullptr; | ||||||||
| 228 | |||||||||
| 229 | V = S->getStepRecurrence(*SE); | ||||||||
| 230 | if (!V) | ||||||||
| 231 | return nullptr; | ||||||||
| 232 | |||||||||
| 233 | // Strip off the size of access multiplication if we are still analyzing the | ||||||||
| 234 | // pointer. | ||||||||
| 235 | if (OrigPtr == Ptr) { | ||||||||
| 236 | if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) { | ||||||||
| 237 | if (M->getOperand(0)->getSCEVType() != scConstant) | ||||||||
| 238 | return nullptr; | ||||||||
| 239 | |||||||||
| 240 | const APInt &APStepVal = cast<SCEVConstant>(M->getOperand(0))->getAPInt(); | ||||||||
| 241 | |||||||||
| 242 | // Huge step value - give up. | ||||||||
| 243 | if (APStepVal.getBitWidth() > 64) | ||||||||
| 244 | return nullptr; | ||||||||
| 245 | |||||||||
| 246 | int64_t StepVal = APStepVal.getSExtValue(); | ||||||||
| 247 | if (PtrAccessSize != StepVal) | ||||||||
| 248 | return nullptr; | ||||||||
| 249 | V = M->getOperand(1); | ||||||||
| 250 | } | ||||||||
| 251 | } | ||||||||
| 252 | |||||||||
| 253 | // Strip off casts. | ||||||||
| 254 | Type *StripedOffRecurrenceCast = nullptr; | ||||||||
| 255 | if (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(V)) { | ||||||||
| 256 | StripedOffRecurrenceCast = C->getType(); | ||||||||
| 257 | V = C->getOperand(); | ||||||||
| 258 | } | ||||||||
| 259 | |||||||||
| 260 | // Look for the loop invariant symbolic value. | ||||||||
| 261 | const SCEVUnknown *U = dyn_cast<SCEVUnknown>(V); | ||||||||
| 262 | if (!U) | ||||||||
| 263 | return nullptr; | ||||||||
| 264 | |||||||||
| 265 | Value *Stride = U->getValue(); | ||||||||
| 266 | if (!Lp->isLoopInvariant(Stride)) | ||||||||
| 267 | return nullptr; | ||||||||
| 268 | |||||||||
| 269 | // If we have stripped off the recurrence cast we have to make sure that we | ||||||||
| 270 | // return the value that is used in this loop so that we can replace it later. | ||||||||
| 271 | if (StripedOffRecurrenceCast) | ||||||||
| 272 | Stride = getUniqueCastUse(Stride, Lp, StripedOffRecurrenceCast); | ||||||||
| 273 | |||||||||
| 274 | return Stride; | ||||||||
| 275 | } | ||||||||
| 276 | |||||||||
| 277 | /// Given a vector and an element number, see if the scalar value is | ||||||||
| 278 | /// already around as a register, for example if it were inserted then extracted | ||||||||
| 279 | /// from the vector. | ||||||||
| 280 | Value *llvm::findScalarElement(Value *V, unsigned EltNo) { | ||||||||
| 281 | assert(V->getType()->isVectorTy() && "Not looking at a vector?")(static_cast <bool> (V->getType()->isVectorTy() && "Not looking at a vector?") ? void (0) : __assert_fail ("V->getType()->isVectorTy() && \"Not looking at a vector?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 281, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 282 | VectorType *VTy = cast<VectorType>(V->getType()); | ||||||||
| 283 | // For fixed-length vector, return undef for out of range access. | ||||||||
| 284 | if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) { | ||||||||
| 285 | unsigned Width = FVTy->getNumElements(); | ||||||||
| 286 | if (EltNo >= Width) | ||||||||
| 287 | return UndefValue::get(FVTy->getElementType()); | ||||||||
| 288 | } | ||||||||
| 289 | |||||||||
| 290 | if (Constant *C = dyn_cast<Constant>(V)) | ||||||||
| 291 | return C->getAggregateElement(EltNo); | ||||||||
| 292 | |||||||||
| 293 | if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) { | ||||||||
| 294 | // If this is an insert to a variable element, we don't know what it is. | ||||||||
| 295 | if (!isa<ConstantInt>(III->getOperand(2))) | ||||||||
| 296 | return nullptr; | ||||||||
| 297 | unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue(); | ||||||||
| 298 | |||||||||
| 299 | // If this is an insert to the element we are looking for, return the | ||||||||
| 300 | // inserted value. | ||||||||
| 301 | if (EltNo == IIElt) | ||||||||
| 302 | return III->getOperand(1); | ||||||||
| 303 | |||||||||
| 304 | // Guard against infinite loop on malformed, unreachable IR. | ||||||||
| 305 | if (III == III->getOperand(0)) | ||||||||
| 306 | return nullptr; | ||||||||
| 307 | |||||||||
| 308 | // Otherwise, the insertelement doesn't modify the value, recurse on its | ||||||||
| 309 | // vector input. | ||||||||
| 310 | return findScalarElement(III->getOperand(0), EltNo); | ||||||||
| 311 | } | ||||||||
| 312 | |||||||||
| 313 | ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V); | ||||||||
| 314 | // Restrict the following transformation to fixed-length vector. | ||||||||
| 315 | if (SVI && isa<FixedVectorType>(SVI->getType())) { | ||||||||
| 316 | unsigned LHSWidth = | ||||||||
| 317 | cast<FixedVectorType>(SVI->getOperand(0)->getType())->getNumElements(); | ||||||||
| 318 | int InEl = SVI->getMaskValue(EltNo); | ||||||||
| 319 | if (InEl < 0) | ||||||||
| 320 | return UndefValue::get(VTy->getElementType()); | ||||||||
| 321 | if (InEl < (int)LHSWidth) | ||||||||
| 322 | return findScalarElement(SVI->getOperand(0), InEl); | ||||||||
| 323 | return findScalarElement(SVI->getOperand(1), InEl - LHSWidth); | ||||||||
| 324 | } | ||||||||
| 325 | |||||||||
| 326 | // Extract a value from a vector add operation with a constant zero. | ||||||||
| 327 | // TODO: Use getBinOpIdentity() to generalize this. | ||||||||
| 328 | Value *Val; Constant *C; | ||||||||
| 329 | if (match(V, m_Add(m_Value(Val), m_Constant(C)))) | ||||||||
| 330 | if (Constant *Elt = C->getAggregateElement(EltNo)) | ||||||||
| 331 | if (Elt->isNullValue()) | ||||||||
| 332 | return findScalarElement(Val, EltNo); | ||||||||
| 333 | |||||||||
| 334 | // Otherwise, we don't know. | ||||||||
| 335 | return nullptr; | ||||||||
| 336 | } | ||||||||
| 337 | |||||||||
| 338 | int llvm::getSplatIndex(ArrayRef<int> Mask) { | ||||||||
| 339 | int SplatIndex = -1; | ||||||||
| 340 | for (int M : Mask) { | ||||||||
| 341 | // Ignore invalid (undefined) mask elements. | ||||||||
| 342 | if (M < 0) | ||||||||
| 343 | continue; | ||||||||
| 344 | |||||||||
| 345 | // There can be only 1 non-negative mask element value if this is a splat. | ||||||||
| 346 | if (SplatIndex != -1 && SplatIndex != M) | ||||||||
| 347 | return -1; | ||||||||
| 348 | |||||||||
| 349 | // Initialize the splat index to the 1st non-negative mask element. | ||||||||
| 350 | SplatIndex = M; | ||||||||
| 351 | } | ||||||||
| 352 | assert((SplatIndex == -1 || SplatIndex >= 0) && "Negative index?")(static_cast <bool> ((SplatIndex == -1 || SplatIndex >= 0) && "Negative index?") ? void (0) : __assert_fail ( "(SplatIndex == -1 || SplatIndex >= 0) && \"Negative index?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 352, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 353 | return SplatIndex; | ||||||||
| 354 | } | ||||||||
| 355 | |||||||||
| 356 | /// Get splat value if the input is a splat vector or return nullptr. | ||||||||
| 357 | /// This function is not fully general. It checks only 2 cases: | ||||||||
| 358 | /// the input value is (1) a splat constant vector or (2) a sequence | ||||||||
| 359 | /// of instructions that broadcasts a scalar at element 0. | ||||||||
| 360 | Value *llvm::getSplatValue(const Value *V) { | ||||||||
| 361 | if (isa<VectorType>(V->getType())) | ||||||||
| 362 | if (auto *C = dyn_cast<Constant>(V)) | ||||||||
| 363 | return C->getSplatValue(); | ||||||||
| 364 | |||||||||
| 365 | // shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...> | ||||||||
| 366 | Value *Splat; | ||||||||
| 367 | if (match(V, | ||||||||
| 368 | m_Shuffle(m_InsertElt(m_Value(), m_Value(Splat), m_ZeroInt()), | ||||||||
| 369 | m_Value(), m_ZeroMask()))) | ||||||||
| 370 | return Splat; | ||||||||
| 371 | |||||||||
| 372 | return nullptr; | ||||||||
| 373 | } | ||||||||
| 374 | |||||||||
| 375 | bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) { | ||||||||
| 376 | assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth")(static_cast <bool> (Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth") ? void (0) : __assert_fail ( "Depth <= MaxAnalysisRecursionDepth && \"Limit Search Depth\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 376, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 377 | |||||||||
| 378 | if (isa<VectorType>(V->getType())) { | ||||||||
| 379 | if (isa<UndefValue>(V)) | ||||||||
| 380 | return true; | ||||||||
| 381 | // FIXME: We can allow undefs, but if Index was specified, we may want to | ||||||||
| 382 | // check that the constant is defined at that index. | ||||||||
| 383 | if (auto *C = dyn_cast<Constant>(V)) | ||||||||
| 384 | return C->getSplatValue() != nullptr; | ||||||||
| 385 | } | ||||||||
| 386 | |||||||||
| 387 | if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) { | ||||||||
| 388 | // FIXME: We can safely allow undefs here. If Index was specified, we will | ||||||||
| 389 | // check that the mask elt is defined at the required index. | ||||||||
| 390 | if (!is_splat(Shuf->getShuffleMask())) | ||||||||
| 391 | return false; | ||||||||
| 392 | |||||||||
| 393 | // Match any index. | ||||||||
| 394 | if (Index == -1) | ||||||||
| 395 | return true; | ||||||||
| 396 | |||||||||
| 397 | // Match a specific element. The mask should be defined at and match the | ||||||||
| 398 | // specified index. | ||||||||
| 399 | return Shuf->getMaskValue(Index) == Index; | ||||||||
| 400 | } | ||||||||
| 401 | |||||||||
| 402 | // The remaining tests are all recursive, so bail out if we hit the limit. | ||||||||
| 403 | if (Depth++ == MaxAnalysisRecursionDepth) | ||||||||
| 404 | return false; | ||||||||
| 405 | |||||||||
| 406 | // If both operands of a binop are splats, the result is a splat. | ||||||||
| 407 | Value *X, *Y, *Z; | ||||||||
| 408 | if (match(V, m_BinOp(m_Value(X), m_Value(Y)))) | ||||||||
| 409 | return isSplatValue(X, Index, Depth) && isSplatValue(Y, Index, Depth); | ||||||||
| 410 | |||||||||
| 411 | // If all operands of a select are splats, the result is a splat. | ||||||||
| 412 | if (match(V, m_Select(m_Value(X), m_Value(Y), m_Value(Z)))) | ||||||||
| 413 | return isSplatValue(X, Index, Depth) && isSplatValue(Y, Index, Depth) && | ||||||||
| 414 | isSplatValue(Z, Index, Depth); | ||||||||
| 415 | |||||||||
| 416 | // TODO: Add support for unary ops (fneg), casts, intrinsics (overflow ops). | ||||||||
| 417 | |||||||||
| 418 | return false; | ||||||||
| 419 | } | ||||||||
| 420 | |||||||||
| 421 | void llvm::narrowShuffleMaskElts(int Scale, ArrayRef<int> Mask, | ||||||||
| 422 | SmallVectorImpl<int> &ScaledMask) { | ||||||||
| 423 | assert(Scale > 0 && "Unexpected scaling factor")(static_cast <bool> (Scale > 0 && "Unexpected scaling factor" ) ? void (0) : __assert_fail ("Scale > 0 && \"Unexpected scaling factor\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 423, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 424 | |||||||||
| 425 | // Fast-path: if no scaling, then it is just a copy. | ||||||||
| 426 | if (Scale == 1) { | ||||||||
| 427 | ScaledMask.assign(Mask.begin(), Mask.end()); | ||||||||
| 428 | return; | ||||||||
| 429 | } | ||||||||
| 430 | |||||||||
| 431 | ScaledMask.clear(); | ||||||||
| 432 | for (int MaskElt : Mask) { | ||||||||
| 433 | if (MaskElt >= 0) { | ||||||||
| 434 | assert(((uint64_t)Scale * MaskElt + (Scale - 1)) <= INT32_MAX &&(static_cast <bool> (((uint64_t)Scale * MaskElt + (Scale - 1)) <= (2147483647) && "Overflowed 32-bits") ? void (0) : __assert_fail ("((uint64_t)Scale * MaskElt + (Scale - 1)) <= INT32_MAX && \"Overflowed 32-bits\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 435, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 435 | "Overflowed 32-bits")(static_cast <bool> (((uint64_t)Scale * MaskElt + (Scale - 1)) <= (2147483647) && "Overflowed 32-bits") ? void (0) : __assert_fail ("((uint64_t)Scale * MaskElt + (Scale - 1)) <= INT32_MAX && \"Overflowed 32-bits\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 435, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 436 | } | ||||||||
| 437 | for (int SliceElt = 0; SliceElt != Scale; ++SliceElt) | ||||||||
| 438 | ScaledMask.push_back(MaskElt < 0 ? MaskElt : Scale * MaskElt + SliceElt); | ||||||||
| 439 | } | ||||||||
| 440 | } | ||||||||
| 441 | |||||||||
| 442 | bool llvm::widenShuffleMaskElts(int Scale, ArrayRef<int> Mask, | ||||||||
| 443 | SmallVectorImpl<int> &ScaledMask) { | ||||||||
| 444 | assert(Scale > 0 && "Unexpected scaling factor")(static_cast <bool> (Scale > 0 && "Unexpected scaling factor" ) ? void (0) : __assert_fail ("Scale > 0 && \"Unexpected scaling factor\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 444, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 445 | |||||||||
| 446 | // Fast-path: if no scaling, then it is just a copy. | ||||||||
| 447 | if (Scale == 1) { | ||||||||
| 448 | ScaledMask.assign(Mask.begin(), Mask.end()); | ||||||||
| 449 | return true; | ||||||||
| 450 | } | ||||||||
| 451 | |||||||||
| 452 | // We must map the original elements down evenly to a type with less elements. | ||||||||
| 453 | int NumElts = Mask.size(); | ||||||||
| 454 | if (NumElts % Scale != 0) | ||||||||
| 455 | return false; | ||||||||
| 456 | |||||||||
| 457 | ScaledMask.clear(); | ||||||||
| 458 | ScaledMask.reserve(NumElts / Scale); | ||||||||
| 459 | |||||||||
| 460 | // Step through the input mask by splitting into Scale-sized slices. | ||||||||
| 461 | do { | ||||||||
| 462 | ArrayRef<int> MaskSlice = Mask.take_front(Scale); | ||||||||
| 463 | assert((int)MaskSlice.size() == Scale && "Expected Scale-sized slice.")(static_cast <bool> ((int)MaskSlice.size() == Scale && "Expected Scale-sized slice.") ? void (0) : __assert_fail ("(int)MaskSlice.size() == Scale && \"Expected Scale-sized slice.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 463, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 464 | |||||||||
| 465 | // The first element of the slice determines how we evaluate this slice. | ||||||||
| 466 | int SliceFront = MaskSlice.front(); | ||||||||
| 467 | if (SliceFront < 0) { | ||||||||
| 468 | // Negative values (undef or other "sentinel" values) must be equal across | ||||||||
| 469 | // the entire slice. | ||||||||
| 470 | if (!is_splat(MaskSlice)) | ||||||||
| 471 | return false; | ||||||||
| 472 | ScaledMask.push_back(SliceFront); | ||||||||
| 473 | } else { | ||||||||
| 474 | // A positive mask element must be cleanly divisible. | ||||||||
| 475 | if (SliceFront % Scale != 0) | ||||||||
| 476 | return false; | ||||||||
| 477 | // Elements of the slice must be consecutive. | ||||||||
| 478 | for (int i = 1; i < Scale; ++i) | ||||||||
| 479 | if (MaskSlice[i] != SliceFront + i) | ||||||||
| 480 | return false; | ||||||||
| 481 | ScaledMask.push_back(SliceFront / Scale); | ||||||||
| 482 | } | ||||||||
| 483 | Mask = Mask.drop_front(Scale); | ||||||||
| 484 | } while (!Mask.empty()); | ||||||||
| 485 | |||||||||
| 486 | assert((int)ScaledMask.size() * Scale == NumElts && "Unexpected scaled mask")(static_cast <bool> ((int)ScaledMask.size() * Scale == NumElts && "Unexpected scaled mask") ? void (0) : __assert_fail ("(int)ScaledMask.size() * Scale == NumElts && \"Unexpected scaled mask\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 486, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 487 | |||||||||
| 488 | // All elements of the original mask can be scaled down to map to the elements | ||||||||
| 489 | // of a mask with wider elements. | ||||||||
| 490 | return true; | ||||||||
| 491 | } | ||||||||
| 492 | |||||||||
| 493 | MapVector<Instruction *, uint64_t> | ||||||||
| 494 | llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB, | ||||||||
| 495 | const TargetTransformInfo *TTI) { | ||||||||
| 496 | |||||||||
| 497 | // DemandedBits will give us every value's live-out bits. But we want | ||||||||
| 498 | // to ensure no extra casts would need to be inserted, so every DAG | ||||||||
| 499 | // of connected values must have the same minimum bitwidth. | ||||||||
| 500 | EquivalenceClasses<Value *> ECs; | ||||||||
| 501 | SmallVector<Value *, 16> Worklist; | ||||||||
| 502 | SmallPtrSet<Value *, 4> Roots; | ||||||||
| 503 | SmallPtrSet<Value *, 16> Visited; | ||||||||
| 504 | DenseMap<Value *, uint64_t> DBits; | ||||||||
| 505 | SmallPtrSet<Instruction *, 4> InstructionSet; | ||||||||
| 506 | MapVector<Instruction *, uint64_t> MinBWs; | ||||||||
| 507 | |||||||||
| 508 | // Determine the roots. We work bottom-up, from truncs or icmps. | ||||||||
| 509 | bool SeenExtFromIllegalType = false; | ||||||||
| 510 | for (auto *BB : Blocks) | ||||||||
| 511 | for (auto &I : *BB) { | ||||||||
| 512 | InstructionSet.insert(&I); | ||||||||
| 513 | |||||||||
| 514 | if (TTI && (isa<ZExtInst>(&I) || isa<SExtInst>(&I)) && | ||||||||
| 515 | !TTI->isTypeLegal(I.getOperand(0)->getType())) | ||||||||
| 516 | SeenExtFromIllegalType = true; | ||||||||
| 517 | |||||||||
| 518 | // Only deal with non-vector integers up to 64-bits wide. | ||||||||
| 519 | if ((isa<TruncInst>(&I) || isa<ICmpInst>(&I)) && | ||||||||
| 520 | !I.getType()->isVectorTy() && | ||||||||
| 521 | I.getOperand(0)->getType()->getScalarSizeInBits() <= 64) { | ||||||||
| 522 | // Don't make work for ourselves. If we know the loaded type is legal, | ||||||||
| 523 | // don't add it to the worklist. | ||||||||
| 524 | if (TTI && isa<TruncInst>(&I) && TTI->isTypeLegal(I.getType())) | ||||||||
| 525 | continue; | ||||||||
| 526 | |||||||||
| 527 | Worklist.push_back(&I); | ||||||||
| 528 | Roots.insert(&I); | ||||||||
| 529 | } | ||||||||
| 530 | } | ||||||||
| 531 | // Early exit. | ||||||||
| 532 | if (Worklist.empty() || (TTI && !SeenExtFromIllegalType)) | ||||||||
| 533 | return MinBWs; | ||||||||
| 534 | |||||||||
| 535 | // Now proceed breadth-first, unioning values together. | ||||||||
| 536 | while (!Worklist.empty()) { | ||||||||
| 537 | Value *Val = Worklist.pop_back_val(); | ||||||||
| 538 | Value *Leader = ECs.getOrInsertLeaderValue(Val); | ||||||||
| 539 | |||||||||
| 540 | if (Visited.count(Val)) | ||||||||
| 541 | continue; | ||||||||
| 542 | Visited.insert(Val); | ||||||||
| 543 | |||||||||
| 544 | // Non-instructions terminate a chain successfully. | ||||||||
| 545 | if (!isa<Instruction>(Val)) | ||||||||
| 546 | continue; | ||||||||
| 547 | Instruction *I = cast<Instruction>(Val); | ||||||||
| 548 | |||||||||
| 549 | // If we encounter a type that is larger than 64 bits, we can't represent | ||||||||
| 550 | // it so bail out. | ||||||||
| 551 | if (DB.getDemandedBits(I).getBitWidth() > 64) | ||||||||
| 552 | return MapVector<Instruction *, uint64_t>(); | ||||||||
| 553 | |||||||||
| 554 | uint64_t V = DB.getDemandedBits(I).getZExtValue(); | ||||||||
| 555 | DBits[Leader] |= V; | ||||||||
| 556 | DBits[I] = V; | ||||||||
| 557 | |||||||||
| 558 | // Casts, loads and instructions outside of our range terminate a chain | ||||||||
| 559 | // successfully. | ||||||||
| 560 | if (isa<SExtInst>(I) || isa<ZExtInst>(I) || isa<LoadInst>(I) || | ||||||||
| 561 | !InstructionSet.count(I)) | ||||||||
| 562 | continue; | ||||||||
| 563 | |||||||||
| 564 | // Unsafe casts terminate a chain unsuccessfully. We can't do anything | ||||||||
| 565 | // useful with bitcasts, ptrtoints or inttoptrs and it'd be unsafe to | ||||||||
| 566 | // transform anything that relies on them. | ||||||||
| 567 | if (isa<BitCastInst>(I) || isa<PtrToIntInst>(I) || isa<IntToPtrInst>(I) || | ||||||||
| 568 | !I->getType()->isIntegerTy()) { | ||||||||
| 569 | DBits[Leader] |= ~0ULL; | ||||||||
| 570 | continue; | ||||||||
| 571 | } | ||||||||
| 572 | |||||||||
| 573 | // We don't modify the types of PHIs. Reductions will already have been | ||||||||
| 574 | // truncated if possible, and inductions' sizes will have been chosen by | ||||||||
| 575 | // indvars. | ||||||||
| 576 | if (isa<PHINode>(I)) | ||||||||
| 577 | continue; | ||||||||
| 578 | |||||||||
| 579 | if (DBits[Leader] == ~0ULL) | ||||||||
| 580 | // All bits demanded, no point continuing. | ||||||||
| 581 | continue; | ||||||||
| 582 | |||||||||
| 583 | for (Value *O : cast<User>(I)->operands()) { | ||||||||
| 584 | ECs.unionSets(Leader, O); | ||||||||
| 585 | Worklist.push_back(O); | ||||||||
| 586 | } | ||||||||
| 587 | } | ||||||||
| 588 | |||||||||
| 589 | // Now we've discovered all values, walk them to see if there are | ||||||||
| 590 | // any users we didn't see. If there are, we can't optimize that | ||||||||
| 591 | // chain. | ||||||||
| 592 | for (auto &I : DBits) | ||||||||
| 593 | for (auto *U : I.first->users()) | ||||||||
| 594 | if (U->getType()->isIntegerTy() && DBits.count(U) == 0) | ||||||||
| 595 | DBits[ECs.getOrInsertLeaderValue(I.first)] |= ~0ULL; | ||||||||
| 596 | |||||||||
| 597 | for (auto I = ECs.begin(), E = ECs.end(); I != E; ++I) { | ||||||||
| 598 | uint64_t LeaderDemandedBits = 0; | ||||||||
| 599 | for (Value *M : llvm::make_range(ECs.member_begin(I), ECs.member_end())) | ||||||||
| 600 | LeaderDemandedBits |= DBits[M]; | ||||||||
| 601 | |||||||||
| 602 | uint64_t MinBW = (sizeof(LeaderDemandedBits) * 8) - | ||||||||
| 603 | llvm::countLeadingZeros(LeaderDemandedBits); | ||||||||
| 604 | // Round up to a power of 2 | ||||||||
| 605 | if (!isPowerOf2_64((uint64_t)MinBW)) | ||||||||
| 606 | MinBW = NextPowerOf2(MinBW); | ||||||||
| 607 | |||||||||
| 608 | // We don't modify the types of PHIs. Reductions will already have been | ||||||||
| 609 | // truncated if possible, and inductions' sizes will have been chosen by | ||||||||
| 610 | // indvars. | ||||||||
| 611 | // If we are required to shrink a PHI, abandon this entire equivalence class. | ||||||||
| 612 | bool Abort = false; | ||||||||
| 613 | for (Value *M : llvm::make_range(ECs.member_begin(I), ECs.member_end())) | ||||||||
| 614 | if (isa<PHINode>(M) && MinBW < M->getType()->getScalarSizeInBits()) { | ||||||||
| 615 | Abort = true; | ||||||||
| 616 | break; | ||||||||
| 617 | } | ||||||||
| 618 | if (Abort) | ||||||||
| 619 | continue; | ||||||||
| 620 | |||||||||
| 621 | for (Value *M : llvm::make_range(ECs.member_begin(I), ECs.member_end())) { | ||||||||
| 622 | if (!isa<Instruction>(M)) | ||||||||
| 623 | continue; | ||||||||
| 624 | Type *Ty = M->getType(); | ||||||||
| 625 | if (Roots.count(M)) | ||||||||
| 626 | Ty = cast<Instruction>(M)->getOperand(0)->getType(); | ||||||||
| 627 | if (MinBW < Ty->getScalarSizeInBits()) | ||||||||
| 628 | MinBWs[cast<Instruction>(M)] = MinBW; | ||||||||
| 629 | } | ||||||||
| 630 | } | ||||||||
| 631 | |||||||||
| 632 | return MinBWs; | ||||||||
| 633 | } | ||||||||
| 634 | |||||||||
| 635 | /// Add all access groups in @p AccGroups to @p List. | ||||||||
| 636 | template <typename ListT> | ||||||||
| 637 | static void addToAccessGroupList(ListT &List, MDNode *AccGroups) { | ||||||||
| 638 | // Interpret an access group as a list containing itself. | ||||||||
| 639 | if (AccGroups->getNumOperands() == 0) { | ||||||||
| 640 | assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group")(static_cast <bool> (isValidAsAccessGroup(AccGroups) && "Node must be an access group") ? void (0) : __assert_fail ( "isValidAsAccessGroup(AccGroups) && \"Node must be an access group\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 640, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 641 | List.insert(AccGroups); | ||||||||
| 642 | return; | ||||||||
| 643 | } | ||||||||
| 644 | |||||||||
| 645 | for (auto &AccGroupListOp : AccGroups->operands()) { | ||||||||
| 646 | auto *Item = cast<MDNode>(AccGroupListOp.get()); | ||||||||
| 647 | assert(isValidAsAccessGroup(Item) && "List item must be an access group")(static_cast <bool> (isValidAsAccessGroup(Item) && "List item must be an access group") ? void (0) : __assert_fail ("isValidAsAccessGroup(Item) && \"List item must be an access group\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 647, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 648 | List.insert(Item); | ||||||||
| 649 | } | ||||||||
| 650 | } | ||||||||
| 651 | |||||||||
| 652 | MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) { | ||||||||
| 653 | if (!AccGroups1) | ||||||||
| 654 | return AccGroups2; | ||||||||
| 655 | if (!AccGroups2) | ||||||||
| 656 | return AccGroups1; | ||||||||
| 657 | if (AccGroups1 == AccGroups2) | ||||||||
| 658 | return AccGroups1; | ||||||||
| 659 | |||||||||
| 660 | SmallSetVector<Metadata *, 4> Union; | ||||||||
| 661 | addToAccessGroupList(Union, AccGroups1); | ||||||||
| 662 | addToAccessGroupList(Union, AccGroups2); | ||||||||
| 663 | |||||||||
| 664 | if (Union.size() == 0) | ||||||||
| 665 | return nullptr; | ||||||||
| 666 | if (Union.size() == 1) | ||||||||
| 667 | return cast<MDNode>(Union.front()); | ||||||||
| 668 | |||||||||
| 669 | LLVMContext &Ctx = AccGroups1->getContext(); | ||||||||
| 670 | return MDNode::get(Ctx, Union.getArrayRef()); | ||||||||
| 671 | } | ||||||||
| 672 | |||||||||
| 673 | MDNode *llvm::intersectAccessGroups(const Instruction *Inst1, | ||||||||
| 674 | const Instruction *Inst2) { | ||||||||
| 675 | bool MayAccessMem1 = Inst1->mayReadOrWriteMemory(); | ||||||||
| 676 | bool MayAccessMem2 = Inst2->mayReadOrWriteMemory(); | ||||||||
| 677 | |||||||||
| 678 | if (!MayAccessMem1 && !MayAccessMem2) | ||||||||
| 679 | return nullptr; | ||||||||
| 680 | if (!MayAccessMem1) | ||||||||
| 681 | return Inst2->getMetadata(LLVMContext::MD_access_group); | ||||||||
| 682 | if (!MayAccessMem2) | ||||||||
| 683 | return Inst1->getMetadata(LLVMContext::MD_access_group); | ||||||||
| 684 | |||||||||
| 685 | MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group); | ||||||||
| 686 | MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group); | ||||||||
| 687 | if (!MD1 || !MD2) | ||||||||
| 688 | return nullptr; | ||||||||
| 689 | if (MD1 == MD2) | ||||||||
| 690 | return MD1; | ||||||||
| 691 | |||||||||
| 692 | // Use set for scalable 'contains' check. | ||||||||
| 693 | SmallPtrSet<Metadata *, 4> AccGroupSet2; | ||||||||
| 694 | addToAccessGroupList(AccGroupSet2, MD2); | ||||||||
| 695 | |||||||||
| 696 | SmallVector<Metadata *, 4> Intersection; | ||||||||
| 697 | if (MD1->getNumOperands() == 0) { | ||||||||
| 698 | assert(isValidAsAccessGroup(MD1) && "Node must be an access group")(static_cast <bool> (isValidAsAccessGroup(MD1) && "Node must be an access group") ? void (0) : __assert_fail ( "isValidAsAccessGroup(MD1) && \"Node must be an access group\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 698, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 699 | if (AccGroupSet2.count(MD1)) | ||||||||
| 700 | Intersection.push_back(MD1); | ||||||||
| 701 | } else { | ||||||||
| 702 | for (const MDOperand &Node : MD1->operands()) { | ||||||||
| 703 | auto *Item = cast<MDNode>(Node.get()); | ||||||||
| 704 | assert(isValidAsAccessGroup(Item) && "List item must be an access group")(static_cast <bool> (isValidAsAccessGroup(Item) && "List item must be an access group") ? void (0) : __assert_fail ("isValidAsAccessGroup(Item) && \"List item must be an access group\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 704, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 705 | if (AccGroupSet2.count(Item)) | ||||||||
| 706 | Intersection.push_back(Item); | ||||||||
| 707 | } | ||||||||
| 708 | } | ||||||||
| 709 | |||||||||
| 710 | if (Intersection.size() == 0) | ||||||||
| 711 | return nullptr; | ||||||||
| 712 | if (Intersection.size() == 1) | ||||||||
| 713 | return cast<MDNode>(Intersection.front()); | ||||||||
| 714 | |||||||||
| 715 | LLVMContext &Ctx = Inst1->getContext(); | ||||||||
| 716 | return MDNode::get(Ctx, Intersection); | ||||||||
| 717 | } | ||||||||
| 718 | |||||||||
| 719 | /// \returns \p I after propagating metadata from \p VL. | ||||||||
| 720 | Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) { | ||||||||
| 721 | if (VL.empty()) | ||||||||
| 722 | return Inst; | ||||||||
| 723 | Instruction *I0 = cast<Instruction>(VL[0]); | ||||||||
| 724 | SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata; | ||||||||
| 725 | I0->getAllMetadataOtherThanDebugLoc(Metadata); | ||||||||
| 726 | |||||||||
| 727 | for (auto Kind : {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, | ||||||||
| 728 | LLVMContext::MD_noalias, LLVMContext::MD_fpmath, | ||||||||
| 729 | LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load, | ||||||||
| 730 | LLVMContext::MD_access_group}) { | ||||||||
| 731 | MDNode *MD = I0->getMetadata(Kind); | ||||||||
| 732 | |||||||||
| 733 | for (int J = 1, E = VL.size(); MD && J != E; ++J) { | ||||||||
| 734 | const Instruction *IJ = cast<Instruction>(VL[J]); | ||||||||
| 735 | MDNode *IMD = IJ->getMetadata(Kind); | ||||||||
| 736 | switch (Kind) { | ||||||||
| 737 | case LLVMContext::MD_tbaa: | ||||||||
| 738 | MD = MDNode::getMostGenericTBAA(MD, IMD); | ||||||||
| 739 | break; | ||||||||
| 740 | case LLVMContext::MD_alias_scope: | ||||||||
| 741 | MD = MDNode::getMostGenericAliasScope(MD, IMD); | ||||||||
| 742 | break; | ||||||||
| 743 | case LLVMContext::MD_fpmath: | ||||||||
| 744 | MD = MDNode::getMostGenericFPMath(MD, IMD); | ||||||||
| 745 | break; | ||||||||
| 746 | case LLVMContext::MD_noalias: | ||||||||
| 747 | case LLVMContext::MD_nontemporal: | ||||||||
| 748 | case LLVMContext::MD_invariant_load: | ||||||||
| 749 | MD = MDNode::intersect(MD, IMD); | ||||||||
| 750 | break; | ||||||||
| 751 | case LLVMContext::MD_access_group: | ||||||||
| 752 | MD = intersectAccessGroups(Inst, IJ); | ||||||||
| 753 | break; | ||||||||
| 754 | default: | ||||||||
| 755 | llvm_unreachable("unhandled metadata")::llvm::llvm_unreachable_internal("unhandled metadata", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 755); | ||||||||
| 756 | } | ||||||||
| 757 | } | ||||||||
| 758 | |||||||||
| 759 | Inst->setMetadata(Kind, MD); | ||||||||
| 760 | } | ||||||||
| 761 | |||||||||
| 762 | return Inst; | ||||||||
| 763 | } | ||||||||
| 764 | |||||||||
| 765 | Constant * | ||||||||
| 766 | llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, | ||||||||
| 767 | const InterleaveGroup<Instruction> &Group) { | ||||||||
| 768 | // All 1's means mask is not needed. | ||||||||
| 769 | if (Group.getNumMembers() == Group.getFactor()) | ||||||||
| 770 | return nullptr; | ||||||||
| 771 | |||||||||
| 772 | // TODO: support reversed access. | ||||||||
| 773 | assert(!Group.isReverse() && "Reversed group not supported.")(static_cast <bool> (!Group.isReverse() && "Reversed group not supported." ) ? void (0) : __assert_fail ("!Group.isReverse() && \"Reversed group not supported.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 773, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 774 | |||||||||
| 775 | SmallVector<Constant *, 16> Mask; | ||||||||
| 776 | for (unsigned i = 0; i < VF; i++) | ||||||||
| 777 | for (unsigned j = 0; j < Group.getFactor(); ++j) { | ||||||||
| 778 | unsigned HasMember = Group.getMember(j) ? 1 : 0; | ||||||||
| 779 | Mask.push_back(Builder.getInt1(HasMember)); | ||||||||
| 780 | } | ||||||||
| 781 | |||||||||
| 782 | return ConstantVector::get(Mask); | ||||||||
| 783 | } | ||||||||
| 784 | |||||||||
| 785 | llvm::SmallVector<int, 16> | ||||||||
| 786 | llvm::createReplicatedMask(unsigned ReplicationFactor, unsigned VF) { | ||||||||
| 787 | SmallVector<int, 16> MaskVec; | ||||||||
| 788 | for (unsigned i = 0; i < VF; i++) | ||||||||
| 789 | for (unsigned j = 0; j < ReplicationFactor; j++) | ||||||||
| 790 | MaskVec.push_back(i); | ||||||||
| 791 | |||||||||
| 792 | return MaskVec; | ||||||||
| 793 | } | ||||||||
| 794 | |||||||||
| 795 | llvm::SmallVector<int, 16> llvm::createInterleaveMask(unsigned VF, | ||||||||
| 796 | unsigned NumVecs) { | ||||||||
| 797 | SmallVector<int, 16> Mask; | ||||||||
| 798 | for (unsigned i = 0; i < VF; i++) | ||||||||
| 799 | for (unsigned j = 0; j < NumVecs; j++) | ||||||||
| 800 | Mask.push_back(j * VF + i); | ||||||||
| 801 | |||||||||
| 802 | return Mask; | ||||||||
| 803 | } | ||||||||
| 804 | |||||||||
| 805 | llvm::SmallVector<int, 16> | ||||||||
| 806 | llvm::createStrideMask(unsigned Start, unsigned Stride, unsigned VF) { | ||||||||
| 807 | SmallVector<int, 16> Mask; | ||||||||
| 808 | for (unsigned i = 0; i < VF; i++) | ||||||||
| 809 | Mask.push_back(Start + i * Stride); | ||||||||
| 810 | |||||||||
| 811 | return Mask; | ||||||||
| 812 | } | ||||||||
| 813 | |||||||||
| 814 | llvm::SmallVector<int, 16> llvm::createSequentialMask(unsigned Start, | ||||||||
| 815 | unsigned NumInts, | ||||||||
| 816 | unsigned NumUndefs) { | ||||||||
| 817 | SmallVector<int, 16> Mask; | ||||||||
| 818 | for (unsigned i = 0; i < NumInts; i++) | ||||||||
| 819 | Mask.push_back(Start + i); | ||||||||
| 820 | |||||||||
| 821 | for (unsigned i = 0; i < NumUndefs; i++) | ||||||||
| 822 | Mask.push_back(-1); | ||||||||
| 823 | |||||||||
| 824 | return Mask; | ||||||||
| 825 | } | ||||||||
| 826 | |||||||||
| 827 | /// A helper function for concatenating vectors. This function concatenates two | ||||||||
| 828 | /// vectors having the same element type. If the second vector has fewer | ||||||||
| 829 | /// elements than the first, it is padded with undefs. | ||||||||
| 830 | static Value *concatenateTwoVectors(IRBuilderBase &Builder, Value *V1, | ||||||||
| 831 | Value *V2) { | ||||||||
| 832 | VectorType *VecTy1 = dyn_cast<VectorType>(V1->getType()); | ||||||||
| 833 | VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType()); | ||||||||
| 834 | assert(VecTy1 && VecTy2 &&(static_cast <bool> (VecTy1 && VecTy2 && VecTy1->getScalarType() == VecTy2->getScalarType() && "Expect two vectors with the same element type") ? void (0) : __assert_fail ("VecTy1 && VecTy2 && VecTy1->getScalarType() == VecTy2->getScalarType() && \"Expect two vectors with the same element type\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 836, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 835 | VecTy1->getScalarType() == VecTy2->getScalarType() &&(static_cast <bool> (VecTy1 && VecTy2 && VecTy1->getScalarType() == VecTy2->getScalarType() && "Expect two vectors with the same element type") ? void (0) : __assert_fail ("VecTy1 && VecTy2 && VecTy1->getScalarType() == VecTy2->getScalarType() && \"Expect two vectors with the same element type\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 836, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 836 | "Expect two vectors with the same element type")(static_cast <bool> (VecTy1 && VecTy2 && VecTy1->getScalarType() == VecTy2->getScalarType() && "Expect two vectors with the same element type") ? void (0) : __assert_fail ("VecTy1 && VecTy2 && VecTy1->getScalarType() == VecTy2->getScalarType() && \"Expect two vectors with the same element type\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 836, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 837 | |||||||||
| 838 | unsigned NumElts1 = cast<FixedVectorType>(VecTy1)->getNumElements(); | ||||||||
| 839 | unsigned NumElts2 = cast<FixedVectorType>(VecTy2)->getNumElements(); | ||||||||
| 840 | assert(NumElts1 >= NumElts2 && "Unexpect the first vector has less elements")(static_cast <bool> (NumElts1 >= NumElts2 && "Unexpect the first vector has less elements") ? void (0) : __assert_fail ("NumElts1 >= NumElts2 && \"Unexpect the first vector has less elements\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 840, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 841 | |||||||||
| 842 | if (NumElts1 > NumElts2) { | ||||||||
| 843 | // Extend with UNDEFs. | ||||||||
| 844 | V2 = Builder.CreateShuffleVector( | ||||||||
| 845 | V2, createSequentialMask(0, NumElts2, NumElts1 - NumElts2)); | ||||||||
| 846 | } | ||||||||
| 847 | |||||||||
| 848 | return Builder.CreateShuffleVector( | ||||||||
| 849 | V1, V2, createSequentialMask(0, NumElts1 + NumElts2, 0)); | ||||||||
| 850 | } | ||||||||
| 851 | |||||||||
| 852 | Value *llvm::concatenateVectors(IRBuilderBase &Builder, | ||||||||
| 853 | ArrayRef<Value *> Vecs) { | ||||||||
| 854 | unsigned NumVecs = Vecs.size(); | ||||||||
| 855 | assert(NumVecs > 1 && "Should be at least two vectors")(static_cast <bool> (NumVecs > 1 && "Should be at least two vectors" ) ? void (0) : __assert_fail ("NumVecs > 1 && \"Should be at least two vectors\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 855, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 856 | |||||||||
| 857 | SmallVector<Value *, 8> ResList; | ||||||||
| 858 | ResList.append(Vecs.begin(), Vecs.end()); | ||||||||
| 859 | do { | ||||||||
| 860 | SmallVector<Value *, 8> TmpList; | ||||||||
| 861 | for (unsigned i = 0; i < NumVecs - 1; i += 2) { | ||||||||
| 862 | Value *V0 = ResList[i], *V1 = ResList[i + 1]; | ||||||||
| 863 | assert((V0->getType() == V1->getType() || i == NumVecs - 2) &&(static_cast <bool> ((V0->getType() == V1->getType () || i == NumVecs - 2) && "Only the last vector may have a different type" ) ? void (0) : __assert_fail ("(V0->getType() == V1->getType() || i == NumVecs - 2) && \"Only the last vector may have a different type\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 864, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 864 | "Only the last vector may have a different type")(static_cast <bool> ((V0->getType() == V1->getType () || i == NumVecs - 2) && "Only the last vector may have a different type" ) ? void (0) : __assert_fail ("(V0->getType() == V1->getType() || i == NumVecs - 2) && \"Only the last vector may have a different type\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 864, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 865 | |||||||||
| 866 | TmpList.push_back(concatenateTwoVectors(Builder, V0, V1)); | ||||||||
| 867 | } | ||||||||
| 868 | |||||||||
| 869 | // Push the last vector if the total number of vectors is odd. | ||||||||
| 870 | if (NumVecs % 2 != 0) | ||||||||
| 871 | TmpList.push_back(ResList[NumVecs - 1]); | ||||||||
| 872 | |||||||||
| 873 | ResList = TmpList; | ||||||||
| 874 | NumVecs = ResList.size(); | ||||||||
| 875 | } while (NumVecs > 1); | ||||||||
| 876 | |||||||||
| 877 | return ResList[0]; | ||||||||
| 878 | } | ||||||||
| 879 | |||||||||
| 880 | bool llvm::maskIsAllZeroOrUndef(Value *Mask) { | ||||||||
| 881 | assert(isa<VectorType>(Mask->getType()) &&(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 885, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 882 | isa<IntegerType>(Mask->getType()->getScalarType()) &&(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 885, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 883 | cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 885, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 884 | 1 &&(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 885, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 885 | "Mask must be a vector of i1")(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 885, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 886 | |||||||||
| 887 | auto *ConstMask = dyn_cast<Constant>(Mask); | ||||||||
| 888 | if (!ConstMask) | ||||||||
| 889 | return false; | ||||||||
| 890 | if (ConstMask->isNullValue() || isa<UndefValue>(ConstMask)) | ||||||||
| 891 | return true; | ||||||||
| 892 | if (isa<ScalableVectorType>(ConstMask->getType())) | ||||||||
| 893 | return false; | ||||||||
| 894 | for (unsigned | ||||||||
| 895 | I = 0, | ||||||||
| 896 | E = cast<FixedVectorType>(ConstMask->getType())->getNumElements(); | ||||||||
| 897 | I != E; ++I) { | ||||||||
| 898 | if (auto *MaskElt = ConstMask->getAggregateElement(I)) | ||||||||
| 899 | if (MaskElt->isNullValue() || isa<UndefValue>(MaskElt)) | ||||||||
| 900 | continue; | ||||||||
| 901 | return false; | ||||||||
| 902 | } | ||||||||
| 903 | return true; | ||||||||
| 904 | } | ||||||||
| 905 | |||||||||
| 906 | bool llvm::maskIsAllOneOrUndef(Value *Mask) { | ||||||||
| 907 | assert(isa<VectorType>(Mask->getType()) &&(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 911, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 908 | isa<IntegerType>(Mask->getType()->getScalarType()) &&(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 911, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 909 | cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 911, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 910 | 1 &&(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 911, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 911 | "Mask must be a vector of i1")(static_cast <bool> (isa<VectorType>(Mask->getType ()) && isa<IntegerType>(Mask->getType()-> getScalarType()) && cast<IntegerType>(Mask-> getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a vector of i1") ? void (0) : __assert_fail ("isa<VectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 911, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 912 | |||||||||
| 913 | auto *ConstMask = dyn_cast<Constant>(Mask); | ||||||||
| 914 | if (!ConstMask) | ||||||||
| 915 | return false; | ||||||||
| 916 | if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask)) | ||||||||
| 917 | return true; | ||||||||
| 918 | if (isa<ScalableVectorType>(ConstMask->getType())) | ||||||||
| 919 | return false; | ||||||||
| 920 | for (unsigned | ||||||||
| 921 | I = 0, | ||||||||
| 922 | E = cast<FixedVectorType>(ConstMask->getType())->getNumElements(); | ||||||||
| 923 | I != E; ++I) { | ||||||||
| 924 | if (auto *MaskElt = ConstMask->getAggregateElement(I)) | ||||||||
| 925 | if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt)) | ||||||||
| 926 | continue; | ||||||||
| 927 | return false; | ||||||||
| 928 | } | ||||||||
| 929 | return true; | ||||||||
| 930 | } | ||||||||
| 931 | |||||||||
| 932 | /// TODO: This is a lot like known bits, but for | ||||||||
| 933 | /// vectors. Is there something we can common this with? | ||||||||
| 934 | APInt llvm::possiblyDemandedEltsInMask(Value *Mask) { | ||||||||
| 935 | assert(isa<FixedVectorType>(Mask->getType()) &&(static_cast <bool> (isa<FixedVectorType>(Mask-> getType()) && isa<IntegerType>(Mask->getType ()->getScalarType()) && cast<IntegerType>(Mask ->getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a fixed width vector of i1") ? void (0) : __assert_fail ("isa<FixedVectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a fixed width vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 939, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 936 | isa<IntegerType>(Mask->getType()->getScalarType()) &&(static_cast <bool> (isa<FixedVectorType>(Mask-> getType()) && isa<IntegerType>(Mask->getType ()->getScalarType()) && cast<IntegerType>(Mask ->getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a fixed width vector of i1") ? void (0) : __assert_fail ("isa<FixedVectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a fixed width vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 939, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 937 | cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==(static_cast <bool> (isa<FixedVectorType>(Mask-> getType()) && isa<IntegerType>(Mask->getType ()->getScalarType()) && cast<IntegerType>(Mask ->getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a fixed width vector of i1") ? void (0) : __assert_fail ("isa<FixedVectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a fixed width vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 939, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 938 | 1 &&(static_cast <bool> (isa<FixedVectorType>(Mask-> getType()) && isa<IntegerType>(Mask->getType ()->getScalarType()) && cast<IntegerType>(Mask ->getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a fixed width vector of i1") ? void (0) : __assert_fail ("isa<FixedVectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a fixed width vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 939, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 939 | "Mask must be a fixed width vector of i1")(static_cast <bool> (isa<FixedVectorType>(Mask-> getType()) && isa<IntegerType>(Mask->getType ()->getScalarType()) && cast<IntegerType>(Mask ->getType()->getScalarType())->getBitWidth() == 1 && "Mask must be a fixed width vector of i1") ? void (0) : __assert_fail ("isa<FixedVectorType>(Mask->getType()) && isa<IntegerType>(Mask->getType()->getScalarType()) && cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1 && \"Mask must be a fixed width vector of i1\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 939, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 940 | |||||||||
| 941 | const unsigned VWidth = | ||||||||
| 942 | cast<FixedVectorType>(Mask->getType())->getNumElements(); | ||||||||
| 943 | APInt DemandedElts = APInt::getAllOnesValue(VWidth); | ||||||||
| 944 | if (auto *CV = dyn_cast<ConstantVector>(Mask)) | ||||||||
| 945 | for (unsigned i = 0; i < VWidth; i++) | ||||||||
| 946 | if (CV->getAggregateElement(i)->isNullValue()) | ||||||||
| 947 | DemandedElts.clearBit(i); | ||||||||
| 948 | return DemandedElts; | ||||||||
| 949 | } | ||||||||
| 950 | |||||||||
| 951 | bool InterleavedAccessInfo::isStrided(int Stride) { | ||||||||
| 952 | unsigned Factor = std::abs(Stride); | ||||||||
| 953 | return Factor >= 2 && Factor <= MaxInterleaveGroupFactor; | ||||||||
| 954 | } | ||||||||
| 955 | |||||||||
| 956 | void InterleavedAccessInfo::collectConstStrideAccesses( | ||||||||
| 957 | MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo, | ||||||||
| 958 | const ValueToValueMap &Strides) { | ||||||||
| 959 | auto &DL = TheLoop->getHeader()->getModule()->getDataLayout(); | ||||||||
| 960 | |||||||||
| 961 | // Since it's desired that the load/store instructions be maintained in | ||||||||
| 962 | // "program order" for the interleaved access analysis, we have to visit the | ||||||||
| 963 | // blocks in the loop in reverse postorder (i.e., in a topological order). | ||||||||
| 964 | // Such an ordering will ensure that any load/store that may be executed | ||||||||
| 965 | // before a second load/store will precede the second load/store in | ||||||||
| 966 | // AccessStrideInfo. | ||||||||
| 967 | LoopBlocksDFS DFS(TheLoop); | ||||||||
| 968 | DFS.perform(LI); | ||||||||
| 969 | for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) | ||||||||
| 970 | for (auto &I : *BB) { | ||||||||
| 971 | Value *Ptr = getLoadStorePointerOperand(&I); | ||||||||
| 972 | if (!Ptr) | ||||||||
| 973 | continue; | ||||||||
| 974 | Type *ElementTy = getLoadStoreType(&I); | ||||||||
| 975 | |||||||||
| 976 | // We don't check wrapping here because we don't know yet if Ptr will be | ||||||||
| 977 | // part of a full group or a group with gaps. Checking wrapping for all | ||||||||
| 978 | // pointers (even those that end up in groups with no gaps) will be overly | ||||||||
| 979 | // conservative. For full groups, wrapping should be ok since if we would | ||||||||
| 980 | // wrap around the address space we would do a memory access at nullptr | ||||||||
| 981 | // even without the transformation. The wrapping checks are therefore | ||||||||
| 982 | // deferred until after we've formed the interleaved groups. | ||||||||
| 983 | int64_t Stride = getPtrStride(PSE, Ptr, TheLoop, Strides, | ||||||||
| 984 | /*Assume=*/true, /*ShouldCheckWrap=*/false); | ||||||||
| 985 | |||||||||
| 986 | const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr); | ||||||||
| 987 | uint64_t Size = DL.getTypeAllocSize(ElementTy); | ||||||||
| 988 | AccessStrideInfo[&I] = StrideDescriptor(Stride, Scev, Size, | ||||||||
| 989 | getLoadStoreAlignment(&I)); | ||||||||
| 990 | } | ||||||||
| 991 | } | ||||||||
| 992 | |||||||||
| 993 | // Analyze interleaved accesses and collect them into interleaved load and | ||||||||
| 994 | // store groups. | ||||||||
| 995 | // | ||||||||
| 996 | // When generating code for an interleaved load group, we effectively hoist all | ||||||||
| 997 | // loads in the group to the location of the first load in program order. When | ||||||||
| 998 | // generating code for an interleaved store group, we sink all stores to the | ||||||||
| 999 | // location of the last store. This code motion can change the order of load | ||||||||
| 1000 | // and store instructions and may break dependences. | ||||||||
| 1001 | // | ||||||||
| 1002 | // The code generation strategy mentioned above ensures that we won't violate | ||||||||
| 1003 | // any write-after-read (WAR) dependences. | ||||||||
| 1004 | // | ||||||||
| 1005 | // E.g., for the WAR dependence: a = A[i]; // (1) | ||||||||
| 1006 | // A[i] = b; // (2) | ||||||||
| 1007 | // | ||||||||
| 1008 | // The store group of (2) is always inserted at or below (2), and the load | ||||||||
| 1009 | // group of (1) is always inserted at or above (1). Thus, the instructions will | ||||||||
| 1010 | // never be reordered. All other dependences are checked to ensure the | ||||||||
| 1011 | // correctness of the instruction reordering. | ||||||||
| 1012 | // | ||||||||
| 1013 | // The algorithm visits all memory accesses in the loop in bottom-up program | ||||||||
| 1014 | // order. Program order is established by traversing the blocks in the loop in | ||||||||
| 1015 | // reverse postorder when collecting the accesses. | ||||||||
| 1016 | // | ||||||||
| 1017 | // We visit the memory accesses in bottom-up order because it can simplify the | ||||||||
| 1018 | // construction of store groups in the presence of write-after-write (WAW) | ||||||||
| 1019 | // dependences. | ||||||||
| 1020 | // | ||||||||
| 1021 | // E.g., for the WAW dependence: A[i] = a; // (1) | ||||||||
| 1022 | // A[i] = b; // (2) | ||||||||
| 1023 | // A[i + 1] = c; // (3) | ||||||||
| 1024 | // | ||||||||
| 1025 | // We will first create a store group with (3) and (2). (1) can't be added to | ||||||||
| 1026 | // this group because it and (2) are dependent. However, (1) can be grouped | ||||||||
| 1027 | // with other accesses that may precede it in program order. Note that a | ||||||||
| 1028 | // bottom-up order does not imply that WAW dependences should not be checked. | ||||||||
| 1029 | void InterleavedAccessInfo::analyzeInterleaving( | ||||||||
| 1030 | bool EnablePredicatedInterleavedMemAccesses) { | ||||||||
| 1031 | LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Analyzing interleaved accesses...\n" ; } } while (false); | ||||||||
| |||||||||
| 1032 | const ValueToValueMap &Strides = LAI->getSymbolicStrides(); | ||||||||
| 1033 | |||||||||
| 1034 | // Holds all accesses with a constant stride. | ||||||||
| 1035 | MapVector<Instruction *, StrideDescriptor> AccessStrideInfo; | ||||||||
| 1036 | collectConstStrideAccesses(AccessStrideInfo, Strides); | ||||||||
| 1037 | |||||||||
| 1038 | if (AccessStrideInfo.empty()) | ||||||||
| 1039 | return; | ||||||||
| 1040 | |||||||||
| 1041 | // Collect the dependences in the loop. | ||||||||
| 1042 | collectDependences(); | ||||||||
| 1043 | |||||||||
| 1044 | // Holds all interleaved store groups temporarily. | ||||||||
| 1045 | SmallSetVector<InterleaveGroup<Instruction> *, 4> StoreGroups; | ||||||||
| 1046 | // Holds all interleaved load groups temporarily. | ||||||||
| 1047 | SmallSetVector<InterleaveGroup<Instruction> *, 4> LoadGroups; | ||||||||
| 1048 | |||||||||
| 1049 | // Search in bottom-up program order for pairs of accesses (A and B) that can | ||||||||
| 1050 | // form interleaved load or store groups. In the algorithm below, access A | ||||||||
| 1051 | // precedes access B in program order. We initialize a group for B in the | ||||||||
| 1052 | // outer loop of the algorithm, and then in the inner loop, we attempt to | ||||||||
| 1053 | // insert each A into B's group if: | ||||||||
| 1054 | // | ||||||||
| 1055 | // 1. A and B have the same stride, | ||||||||
| 1056 | // 2. A and B have the same memory object size, and | ||||||||
| 1057 | // 3. A belongs in B's group according to its distance from B. | ||||||||
| 1058 | // | ||||||||
| 1059 | // Special care is taken to ensure group formation will not break any | ||||||||
| 1060 | // dependences. | ||||||||
| 1061 | for (auto BI = AccessStrideInfo.rbegin(), E = AccessStrideInfo.rend(); | ||||||||
| 1062 | BI != E; ++BI) { | ||||||||
| 1063 | Instruction *B = BI->first; | ||||||||
| 1064 | StrideDescriptor DesB = BI->second; | ||||||||
| 1065 | |||||||||
| 1066 | // Initialize a group for B if it has an allowable stride. Even if we don't | ||||||||
| 1067 | // create a group for B, we continue with the bottom-up algorithm to ensure | ||||||||
| 1068 | // we don't break any of B's dependences. | ||||||||
| 1069 | InterleaveGroup<Instruction> *Group = nullptr; | ||||||||
| 1070 | if (isStrided(DesB.Stride) && | ||||||||
| 1071 | (!isPredicated(B->getParent()) || EnablePredicatedInterleavedMemAccesses)) { | ||||||||
| 1072 | Group = getInterleaveGroup(B); | ||||||||
| 1073 | if (!Group) { | ||||||||
| 1074 | LLVM_DEBUG(dbgs() << "LV: Creating an interleave group with:" << *Bdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Creating an interleave group with:" << *B << '\n'; } } while (false) | ||||||||
| 1075 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Creating an interleave group with:" << *B << '\n'; } } while (false); | ||||||||
| 1076 | Group = createInterleaveGroup(B, DesB.Stride, DesB.Alignment); | ||||||||
| 1077 | } | ||||||||
| 1078 | if (B->mayWriteToMemory()) | ||||||||
| 1079 | StoreGroups.insert(Group); | ||||||||
| 1080 | else | ||||||||
| 1081 | LoadGroups.insert(Group); | ||||||||
| 1082 | } | ||||||||
| 1083 | |||||||||
| 1084 | for (auto AI = std::next(BI); AI != E; ++AI) { | ||||||||
| 1085 | Instruction *A = AI->first; | ||||||||
| 1086 | StrideDescriptor DesA = AI->second; | ||||||||
| 1087 | |||||||||
| 1088 | // Our code motion strategy implies that we can't have dependences | ||||||||
| 1089 | // between accesses in an interleaved group and other accesses located | ||||||||
| 1090 | // between the first and last member of the group. Note that this also | ||||||||
| 1091 | // means that a group can't have more than one member at a given offset. | ||||||||
| 1092 | // The accesses in a group can have dependences with other accesses, but | ||||||||
| 1093 | // we must ensure we don't extend the boundaries of the group such that | ||||||||
| 1094 | // we encompass those dependent accesses. | ||||||||
| 1095 | // | ||||||||
| 1096 | // For example, assume we have the sequence of accesses shown below in a | ||||||||
| 1097 | // stride-2 loop: | ||||||||
| 1098 | // | ||||||||
| 1099 | // (1, 2) is a group | A[i] = a; // (1) | ||||||||
| 1100 | // | A[i-1] = b; // (2) | | ||||||||
| 1101 | // A[i-3] = c; // (3) | ||||||||
| 1102 | // A[i] = d; // (4) | (2, 4) is not a group | ||||||||
| 1103 | // | ||||||||
| 1104 | // Because accesses (2) and (3) are dependent, we can group (2) with (1) | ||||||||
| 1105 | // but not with (4). If we did, the dependent access (3) would be within | ||||||||
| 1106 | // the boundaries of the (2, 4) group. | ||||||||
| 1107 | if (!canReorderMemAccessesForInterleavedGroups(&*AI, &*BI)) { | ||||||||
| 1108 | // If a dependence exists and A is already in a group, we know that A | ||||||||
| 1109 | // must be a store since A precedes B and WAR dependences are allowed. | ||||||||
| 1110 | // Thus, A would be sunk below B. We release A's group to prevent this | ||||||||
| 1111 | // illegal code motion. A will then be free to form another group with | ||||||||
| 1112 | // instructions that precede it. | ||||||||
| 1113 | if (isInterleaved(A)) { | ||||||||
| 1114 | InterleaveGroup<Instruction> *StoreGroup = getInterleaveGroup(A); | ||||||||
| 1115 | |||||||||
| 1116 | LLVM_DEBUG(dbgs() << "LV: Invalidated store group due to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidated store group due to " "dependence between " << *A << " and "<< * B << '\n'; } } while (false) | ||||||||
| 1117 | "dependence between " << *A << " and "<< *B << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidated store group due to " "dependence between " << *A << " and "<< * B << '\n'; } } while (false); | ||||||||
| 1118 | |||||||||
| 1119 | StoreGroups.remove(StoreGroup); | ||||||||
| 1120 | releaseGroup(StoreGroup); | ||||||||
| 1121 | } | ||||||||
| 1122 | |||||||||
| 1123 | // If a dependence exists and A is not already in a group (or it was | ||||||||
| 1124 | // and we just released it), B might be hoisted above A (if B is a | ||||||||
| 1125 | // load) or another store might be sunk below A (if B is a store). In | ||||||||
| 1126 | // either case, we can't add additional instructions to B's group. B | ||||||||
| 1127 | // will only form a group with instructions that it precedes. | ||||||||
| 1128 | break; | ||||||||
| 1129 | } | ||||||||
| 1130 | |||||||||
| 1131 | // At this point, we've checked for illegal code motion. If either A or B | ||||||||
| 1132 | // isn't strided, there's nothing left to do. | ||||||||
| 1133 | if (!isStrided(DesA.Stride) || !isStrided(DesB.Stride)) | ||||||||
| 1134 | continue; | ||||||||
| 1135 | |||||||||
| 1136 | // Ignore A if it's already in a group or isn't the same kind of memory | ||||||||
| 1137 | // operation as B. | ||||||||
| 1138 | // Note that mayReadFromMemory() isn't mutually exclusive to | ||||||||
| 1139 | // mayWriteToMemory in the case of atomic loads. We shouldn't see those | ||||||||
| 1140 | // here, canVectorizeMemory() should have returned false - except for the | ||||||||
| 1141 | // case we asked for optimization remarks. | ||||||||
| 1142 | if (isInterleaved(A) || | ||||||||
| 1143 | (A->mayReadFromMemory() != B->mayReadFromMemory()) || | ||||||||
| 1144 | (A->mayWriteToMemory() != B->mayWriteToMemory())) | ||||||||
| 1145 | continue; | ||||||||
| 1146 | |||||||||
| 1147 | // Check rules 1 and 2. Ignore A if its stride or size is different from | ||||||||
| 1148 | // that of B. | ||||||||
| 1149 | if (DesA.Stride != DesB.Stride || DesA.Size != DesB.Size) | ||||||||
| 1150 | continue; | ||||||||
| 1151 | |||||||||
| 1152 | // Ignore A if the memory object of A and B don't belong to the same | ||||||||
| 1153 | // address space | ||||||||
| 1154 | if (getLoadStoreAddressSpace(A) != getLoadStoreAddressSpace(B)) | ||||||||
| 1155 | continue; | ||||||||
| 1156 | |||||||||
| 1157 | // Calculate the distance from A to B. | ||||||||
| 1158 | const SCEVConstant *DistToB = dyn_cast<SCEVConstant>( | ||||||||
| 1159 | PSE.getSE()->getMinusSCEV(DesA.Scev, DesB.Scev)); | ||||||||
| 1160 | if (!DistToB
| ||||||||
| 1161 | continue; | ||||||||
| 1162 | int64_t DistanceToB = DistToB->getAPInt().getSExtValue(); | ||||||||
| 1163 | |||||||||
| 1164 | // Check rule 3. Ignore A if its distance to B is not a multiple of the | ||||||||
| 1165 | // size. | ||||||||
| 1166 | if (DistanceToB % static_cast<int64_t>(DesB.Size)) | ||||||||
| 1167 | continue; | ||||||||
| 1168 | |||||||||
| 1169 | // All members of a predicated interleave-group must have the same predicate, | ||||||||
| 1170 | // and currently must reside in the same BB. | ||||||||
| 1171 | BasicBlock *BlockA = A->getParent(); | ||||||||
| 1172 | BasicBlock *BlockB = B->getParent(); | ||||||||
| 1173 | if ((isPredicated(BlockA) || isPredicated(BlockB)) && | ||||||||
| 1174 | (!EnablePredicatedInterleavedMemAccesses || BlockA != BlockB)) | ||||||||
| 1175 | continue; | ||||||||
| 1176 | |||||||||
| 1177 | // The index of A is the index of B plus A's distance to B in multiples | ||||||||
| 1178 | // of the size. | ||||||||
| 1179 | int IndexA = | ||||||||
| 1180 | Group->getIndex(B) + DistanceToB / static_cast<int64_t>(DesB.Size); | ||||||||
| |||||||||
| 1181 | |||||||||
| 1182 | // Try to insert A into B's group. | ||||||||
| 1183 | if (Group->insertMember(A, IndexA, DesA.Alignment)) { | ||||||||
| 1184 | LLVM_DEBUG(dbgs() << "LV: Inserted:" << *A << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Inserted:" << * A << '\n' << " into the interleave group with" << *B << '\n'; } } while (false) | ||||||||
| 1185 | << " into the interleave group with" << *Bdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Inserted:" << * A << '\n' << " into the interleave group with" << *B << '\n'; } } while (false) | ||||||||
| 1186 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Inserted:" << * A << '\n' << " into the interleave group with" << *B << '\n'; } } while (false); | ||||||||
| 1187 | InterleaveGroupMap[A] = Group; | ||||||||
| 1188 | |||||||||
| 1189 | // Set the first load in program order as the insert position. | ||||||||
| 1190 | if (A->mayReadFromMemory()) | ||||||||
| 1191 | Group->setInsertPos(A); | ||||||||
| 1192 | } | ||||||||
| 1193 | } // Iteration over A accesses. | ||||||||
| 1194 | } // Iteration over B accesses. | ||||||||
| 1195 | |||||||||
| 1196 | // Remove interleaved store groups with gaps. | ||||||||
| 1197 | for (auto *Group : StoreGroups) | ||||||||
| 1198 | if (Group->getNumMembers() != Group->getFactor()) { | ||||||||
| 1199 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved store group due " "to gaps.\n"; } } while (false) | ||||||||
| 1200 | dbgs() << "LV: Invalidate candidate interleaved store group due "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved store group due " "to gaps.\n"; } } while (false) | ||||||||
| 1201 | "to gaps.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved store group due " "to gaps.\n"; } } while (false); | ||||||||
| 1202 | releaseGroup(Group); | ||||||||
| 1203 | } | ||||||||
| 1204 | // Remove interleaved groups with gaps (currently only loads) whose memory | ||||||||
| 1205 | // accesses may wrap around. We have to revisit the getPtrStride analysis, | ||||||||
| 1206 | // this time with ShouldCheckWrap=true, since collectConstStrideAccesses does | ||||||||
| 1207 | // not check wrapping (see documentation there). | ||||||||
| 1208 | // FORNOW we use Assume=false; | ||||||||
| 1209 | // TODO: Change to Assume=true but making sure we don't exceed the threshold | ||||||||
| 1210 | // of runtime SCEV assumptions checks (thereby potentially failing to | ||||||||
| 1211 | // vectorize altogether). | ||||||||
| 1212 | // Additional optional optimizations: | ||||||||
| 1213 | // TODO: If we are peeling the loop and we know that the first pointer doesn't | ||||||||
| 1214 | // wrap then we can deduce that all pointers in the group don't wrap. | ||||||||
| 1215 | // This means that we can forcefully peel the loop in order to only have to | ||||||||
| 1216 | // check the first pointer for no-wrap. When we'll change to use Assume=true | ||||||||
| 1217 | // we'll only need at most one runtime check per interleaved group. | ||||||||
| 1218 | for (auto *Group : LoadGroups) { | ||||||||
| 1219 | // Case 1: A full group. Can Skip the checks; For full groups, if the wide | ||||||||
| 1220 | // load would wrap around the address space we would do a memory access at | ||||||||
| 1221 | // nullptr even without the transformation. | ||||||||
| 1222 | if (Group->getNumMembers() == Group->getFactor()) | ||||||||
| 1223 | continue; | ||||||||
| 1224 | |||||||||
| 1225 | // Case 2: If first and last members of the group don't wrap this implies | ||||||||
| 1226 | // that all the pointers in the group don't wrap. | ||||||||
| 1227 | // So we check only group member 0 (which is always guaranteed to exist), | ||||||||
| 1228 | // and group member Factor - 1; If the latter doesn't exist we rely on | ||||||||
| 1229 | // peeling (if it is a non-reversed accsess -- see Case 3). | ||||||||
| 1230 | Value *FirstMemberPtr = getLoadStorePointerOperand(Group->getMember(0)); | ||||||||
| 1231 | if (!getPtrStride(PSE, FirstMemberPtr, TheLoop, Strides, /*Assume=*/false, | ||||||||
| 1232 | /*ShouldCheckWrap=*/true)) { | ||||||||
| 1233 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "first group member potentially pointer-wrapping.\n"; } } while (false) | ||||||||
| 1234 | dbgs() << "LV: Invalidate candidate interleaved group due to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "first group member potentially pointer-wrapping.\n"; } } while (false) | ||||||||
| 1235 | "first group member potentially pointer-wrapping.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "first group member potentially pointer-wrapping.\n"; } } while (false); | ||||||||
| 1236 | releaseGroup(Group); | ||||||||
| 1237 | continue; | ||||||||
| 1238 | } | ||||||||
| 1239 | Instruction *LastMember = Group->getMember(Group->getFactor() - 1); | ||||||||
| 1240 | if (LastMember) { | ||||||||
| 1241 | Value *LastMemberPtr = getLoadStorePointerOperand(LastMember); | ||||||||
| 1242 | if (!getPtrStride(PSE, LastMemberPtr, TheLoop, Strides, /*Assume=*/false, | ||||||||
| 1243 | /*ShouldCheckWrap=*/true)) { | ||||||||
| 1244 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "last group member potentially pointer-wrapping.\n"; } } while (false) | ||||||||
| 1245 | dbgs() << "LV: Invalidate candidate interleaved group due to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "last group member potentially pointer-wrapping.\n"; } } while (false) | ||||||||
| 1246 | "last group member potentially pointer-wrapping.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "last group member potentially pointer-wrapping.\n"; } } while (false); | ||||||||
| 1247 | releaseGroup(Group); | ||||||||
| 1248 | } | ||||||||
| 1249 | } else { | ||||||||
| 1250 | // Case 3: A non-reversed interleaved load group with gaps: We need | ||||||||
| 1251 | // to execute at least one scalar epilogue iteration. This will ensure | ||||||||
| 1252 | // we don't speculatively access memory out-of-bounds. We only need | ||||||||
| 1253 | // to look for a member at index factor - 1, since every group must have | ||||||||
| 1254 | // a member at index zero. | ||||||||
| 1255 | if (Group->isReverse()) { | ||||||||
| 1256 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "a reverse access with gaps.\n"; } } while (false) | ||||||||
| 1257 | dbgs() << "LV: Invalidate candidate interleaved group due to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "a reverse access with gaps.\n"; } } while (false) | ||||||||
| 1258 | "a reverse access with gaps.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to " "a reverse access with gaps.\n"; } } while (false); | ||||||||
| 1259 | releaseGroup(Group); | ||||||||
| 1260 | continue; | ||||||||
| 1261 | } | ||||||||
| 1262 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Interleaved group requires epilogue iteration.\n" ; } } while (false) | ||||||||
| 1263 | dbgs() << "LV: Interleaved group requires epilogue iteration.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Interleaved group requires epilogue iteration.\n" ; } } while (false); | ||||||||
| 1264 | RequiresScalarEpilogue = true; | ||||||||
| 1265 | } | ||||||||
| 1266 | } | ||||||||
| 1267 | } | ||||||||
| 1268 | |||||||||
| 1269 | void InterleavedAccessInfo::invalidateGroupsRequiringScalarEpilogue() { | ||||||||
| 1270 | // If no group had triggered the requirement to create an epilogue loop, | ||||||||
| 1271 | // there is nothing to do. | ||||||||
| 1272 | if (!requiresScalarEpilogue()) | ||||||||
| 1273 | return; | ||||||||
| 1274 | |||||||||
| 1275 | bool ReleasedGroup = false; | ||||||||
| 1276 | // Release groups requiring scalar epilogues. Note that this also removes them | ||||||||
| 1277 | // from InterleaveGroups. | ||||||||
| 1278 | for (auto *Group : make_early_inc_range(InterleaveGroups)) { | ||||||||
| 1279 | if (!Group->requiresScalarEpilogue()) | ||||||||
| 1280 | continue; | ||||||||
| 1281 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to gaps that " "require a scalar epilogue (not allowed under optsize) and cannot " "be masked (not enabled). \n"; } } while (false) | ||||||||
| 1282 | dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to gaps that " "require a scalar epilogue (not allowed under optsize) and cannot " "be masked (not enabled). \n"; } } while (false) | ||||||||
| 1283 | << "LV: Invalidate candidate interleaved group due to gaps that "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to gaps that " "require a scalar epilogue (not allowed under optsize) and cannot " "be masked (not enabled). \n"; } } while (false) | ||||||||
| 1284 | "require a scalar epilogue (not allowed under optsize) and cannot "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to gaps that " "require a scalar epilogue (not allowed under optsize) and cannot " "be masked (not enabled). \n"; } } while (false) | ||||||||
| 1285 | "be masked (not enabled). \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "LV: Invalidate candidate interleaved group due to gaps that " "require a scalar epilogue (not allowed under optsize) and cannot " "be masked (not enabled). \n"; } } while (false); | ||||||||
| 1286 | releaseGroup(Group); | ||||||||
| 1287 | ReleasedGroup = true; | ||||||||
| 1288 | } | ||||||||
| 1289 | assert(ReleasedGroup && "At least one group must be invalidated, as a "(static_cast <bool> (ReleasedGroup && "At least one group must be invalidated, as a " "scalar epilogue was required") ? void (0) : __assert_fail ( "ReleasedGroup && \"At least one group must be invalidated, as a \" \"scalar epilogue was required\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 1290, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1290 | "scalar epilogue was required")(static_cast <bool> (ReleasedGroup && "At least one group must be invalidated, as a " "scalar epilogue was required") ? void (0) : __assert_fail ( "ReleasedGroup && \"At least one group must be invalidated, as a \" \"scalar epilogue was required\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 1290, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1291 | (void)ReleasedGroup; | ||||||||
| 1292 | RequiresScalarEpilogue = false; | ||||||||
| 1293 | } | ||||||||
| 1294 | |||||||||
| 1295 | template <typename InstT> | ||||||||
| 1296 | void InterleaveGroup<InstT>::addMetadata(InstT *NewInst) const { | ||||||||
| 1297 | llvm_unreachable("addMetadata can only be used for Instruction")::llvm::llvm_unreachable_internal("addMetadata can only be used for Instruction" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 1297); | ||||||||
| 1298 | } | ||||||||
| 1299 | |||||||||
| 1300 | namespace llvm { | ||||||||
| 1301 | template <> | ||||||||
| 1302 | void InterleaveGroup<Instruction>::addMetadata(Instruction *NewInst) const { | ||||||||
| 1303 | SmallVector<Value *, 4> VL; | ||||||||
| 1304 | std::transform(Members.begin(), Members.end(), std::back_inserter(VL), | ||||||||
| 1305 | [](std::pair<int, Instruction *> p) { return p.second; }); | ||||||||
| 1306 | propagateMetadata(NewInst, VL); | ||||||||
| 1307 | } | ||||||||
| 1308 | } | ||||||||
| 1309 | |||||||||
| 1310 | std::string VFABI::mangleTLIVectorName(StringRef VectorName, | ||||||||
| 1311 | StringRef ScalarName, unsigned numArgs, | ||||||||
| 1312 | ElementCount VF) { | ||||||||
| 1313 | SmallString<256> Buffer; | ||||||||
| 1314 | llvm::raw_svector_ostream Out(Buffer); | ||||||||
| 1315 | Out << "_ZGV" << VFABI::_LLVM_ << "N"; | ||||||||
| 1316 | if (VF.isScalable()) | ||||||||
| 1317 | Out << 'x'; | ||||||||
| 1318 | else | ||||||||
| 1319 | Out << VF.getFixedValue(); | ||||||||
| 1320 | for (unsigned I = 0; I < numArgs; ++I) | ||||||||
| 1321 | Out << "v"; | ||||||||
| 1322 | Out << "_" << ScalarName << "(" << VectorName << ")"; | ||||||||
| 1323 | return std::string(Out.str()); | ||||||||
| 1324 | } | ||||||||
| 1325 | |||||||||
| 1326 | void VFABI::getVectorVariantNames( | ||||||||
| 1327 | const CallInst &CI, SmallVectorImpl<std::string> &VariantMappings) { | ||||||||
| 1328 | const StringRef S = | ||||||||
| 1329 | CI.getAttribute(AttributeList::FunctionIndex, VFABI::MappingsAttrName) | ||||||||
| 1330 | .getValueAsString(); | ||||||||
| 1331 | if (S.empty()) | ||||||||
| 1332 | return; | ||||||||
| 1333 | |||||||||
| 1334 | SmallVector<StringRef, 8> ListAttr; | ||||||||
| 1335 | S.split(ListAttr, ","); | ||||||||
| 1336 | |||||||||
| 1337 | for (auto &S : SetVector<StringRef>(ListAttr.begin(), ListAttr.end())) { | ||||||||
| 1338 | #ifndef NDEBUG | ||||||||
| 1339 | LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << S << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vectorutils")) { dbgs() << "VFABI: adding mapping '" << S << "'\n"; } } while (false); | ||||||||
| 1340 | Optional<VFInfo> Info = VFABI::tryDemangleForVFABI(S, *(CI.getModule())); | ||||||||
| 1341 | assert(Info.hasValue() && "Invalid name for a VFABI variant.")(static_cast <bool> (Info.hasValue() && "Invalid name for a VFABI variant." ) ? void (0) : __assert_fail ("Info.hasValue() && \"Invalid name for a VFABI variant.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 1341, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1342 | assert(CI.getModule()->getFunction(Info.getValue().VectorName) &&(static_cast <bool> (CI.getModule()->getFunction(Info .getValue().VectorName) && "Vector function is missing." ) ? void (0) : __assert_fail ("CI.getModule()->getFunction(Info.getValue().VectorName) && \"Vector function is missing.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 1343, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1343 | "Vector function is missing.")(static_cast <bool> (CI.getModule()->getFunction(Info .getValue().VectorName) && "Vector function is missing." ) ? void (0) : __assert_fail ("CI.getModule()->getFunction(Info.getValue().VectorName) && \"Vector function is missing.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 1343, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1344 | #endif | ||||||||
| 1345 | VariantMappings.push_back(std::string(S)); | ||||||||
| 1346 | } | ||||||||
| 1347 | } | ||||||||
| 1348 | |||||||||
| 1349 | bool VFShape::hasValidParameterList() const { | ||||||||
| 1350 | for (unsigned Pos = 0, NumParams = Parameters.size(); Pos < NumParams; | ||||||||
| 1351 | ++Pos) { | ||||||||
| 1352 | assert(Parameters[Pos].ParamPos == Pos && "Broken parameter list.")(static_cast <bool> (Parameters[Pos].ParamPos == Pos && "Broken parameter list.") ? void (0) : __assert_fail ("Parameters[Pos].ParamPos == Pos && \"Broken parameter list.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Analysis/VectorUtils.cpp" , 1352, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1353 | |||||||||
| 1354 | switch (Parameters[Pos].ParamKind) { | ||||||||
| 1355 | default: // Nothing to check. | ||||||||
| 1356 | break; | ||||||||
| 1357 | case VFParamKind::OMP_Linear: | ||||||||
| 1358 | case VFParamKind::OMP_LinearRef: | ||||||||
| 1359 | case VFParamKind::OMP_LinearVal: | ||||||||
| 1360 | case VFParamKind::OMP_LinearUVal: | ||||||||
| 1361 | // Compile time linear steps must be non-zero. | ||||||||
| 1362 | if (Parameters[Pos].LinearStepOrPos == 0) | ||||||||
| 1363 | return false; | ||||||||
| 1364 | break; | ||||||||
| 1365 | case VFParamKind::OMP_LinearPos: | ||||||||
| 1366 | case VFParamKind::OMP_LinearRefPos: | ||||||||
| 1367 | case VFParamKind::OMP_LinearValPos: | ||||||||
| 1368 | case VFParamKind::OMP_LinearUValPos: | ||||||||
| 1369 | // The runtime linear step must be referring to some other | ||||||||
| 1370 | // parameters in the signature. | ||||||||
| 1371 | if (Parameters[Pos].LinearStepOrPos >= int(NumParams)) | ||||||||
| 1372 | return false; | ||||||||
| 1373 | // The linear step parameter must be marked as uniform. | ||||||||
| 1374 | if (Parameters[Parameters[Pos].LinearStepOrPos].ParamKind != | ||||||||
| 1375 | VFParamKind::OMP_Uniform) | ||||||||
| 1376 | return false; | ||||||||
| 1377 | // The linear step parameter can't point at itself. | ||||||||
| 1378 | if (Parameters[Pos].LinearStepOrPos == int(Pos)) | ||||||||
| 1379 | return false; | ||||||||
| 1380 | break; | ||||||||
| 1381 | case VFParamKind::GlobalPredicate: | ||||||||
| 1382 | // The global predicate must be the unique. Can be placed anywhere in the | ||||||||
| 1383 | // signature. | ||||||||
| 1384 | for (unsigned NextPos = Pos + 1; NextPos < NumParams; ++NextPos) | ||||||||
| 1385 | if (Parameters[NextPos].ParamKind == VFParamKind::GlobalPredicate) | ||||||||
| 1386 | return false; | ||||||||
| 1387 | break; | ||||||||
| 1388 | } | ||||||||
| 1389 | } | ||||||||
| 1390 | return true; | ||||||||
| 1391 | } |
| 1 | // Iterators -*- C++ -*- |
| 2 | |
| 3 | // Copyright (C) 2001-2020 Free Software Foundation, Inc. |
| 4 | // |
| 5 | // This file is part of the GNU ISO C++ Library. This library is free |
| 6 | // software; you can redistribute it and/or modify it under the |
| 7 | // terms of the GNU General Public License as published by the |
| 8 | // Free Software Foundation; either version 3, or (at your option) |
| 9 | // any later version. |
| 10 | |
| 11 | // This library is distributed in the hope that it will be useful, |
| 12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | // GNU General Public License for more details. |
| 15 | |
| 16 | // Under Section 7 of GPL version 3, you are granted additional |
| 17 | // permissions described in the GCC Runtime Library Exception, version |
| 18 | // 3.1, as published by the Free Software Foundation. |
| 19 | |
| 20 | // You should have received a copy of the GNU General Public License and |
| 21 | // a copy of the GCC Runtime Library Exception along with this program; |
| 22 | // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
| 23 | // <http://www.gnu.org/licenses/>. |
| 24 | |
| 25 | /* |
| 26 | * |
| 27 | * Copyright (c) 1994 |
| 28 | * Hewlett-Packard Company |
| 29 | * |
| 30 | * Permission to use, copy, modify, distribute and sell this software |
| 31 | * and its documentation for any purpose is hereby granted without fee, |
| 32 | * provided that the above copyright notice appear in all copies and |
| 33 | * that both that copyright notice and this permission notice appear |
| 34 | * in supporting documentation. Hewlett-Packard Company makes no |
| 35 | * representations about the suitability of this software for any |
| 36 | * purpose. It is provided "as is" without express or implied warranty. |
| 37 | * |
| 38 | * |
| 39 | * Copyright (c) 1996-1998 |
| 40 | * Silicon Graphics Computer Systems, Inc. |
| 41 | * |
| 42 | * Permission to use, copy, modify, distribute and sell this software |
| 43 | * and its documentation for any purpose is hereby granted without fee, |
| 44 | * provided that the above copyright notice appear in all copies and |
| 45 | * that both that copyright notice and this permission notice appear |
| 46 | * in supporting documentation. Silicon Graphics makes no |
| 47 | * representations about the suitability of this software for any |
| 48 | * purpose. It is provided "as is" without express or implied warranty. |
| 49 | */ |
| 50 | |
| 51 | /** @file bits/stl_iterator.h |
| 52 | * This is an internal header file, included by other library headers. |
| 53 | * Do not attempt to use it directly. @headername{iterator} |
| 54 | * |
| 55 | * This file implements reverse_iterator, back_insert_iterator, |
| 56 | * front_insert_iterator, insert_iterator, __normal_iterator, and their |
| 57 | * supporting functions and overloaded operators. |
| 58 | */ |
| 59 | |
| 60 | #ifndef _STL_ITERATOR_H1 |
| 61 | #define _STL_ITERATOR_H1 1 |
| 62 | |
| 63 | #include <bits/cpp_type_traits.h> |
| 64 | #include <ext/type_traits.h> |
| 65 | #include <bits/move.h> |
| 66 | #include <bits/ptr_traits.h> |
| 67 | |
| 68 | #if __cplusplus201402L >= 201103L |
| 69 | # include <type_traits> |
| 70 | #endif |
| 71 | |
| 72 | #if __cplusplus201402L > 201703L |
| 73 | # define __cpp_lib_array_constexpr 201811L |
| 74 | # define __cpp_lib_constexpr_iterator 201811L |
| 75 | #elif __cplusplus201402L == 201703L |
| 76 | # define __cpp_lib_array_constexpr 201803L |
| 77 | #endif |
| 78 | |
| 79 | #if __cplusplus201402L > 201703L |
| 80 | # include <compare> |
| 81 | # include <new> |
| 82 | # include <bits/iterator_concepts.h> |
| 83 | #endif |
| 84 | |
| 85 | namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default"))) |
| 86 | { |
| 87 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
| 88 | |
| 89 | /** |
| 90 | * @addtogroup iterators |
| 91 | * @{ |
| 92 | */ |
| 93 | |
| 94 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 95 | namespace __detail |
| 96 | { |
| 97 | // Weaken iterator_category _Cat to _Limit if it is derived from that, |
| 98 | // otherwise use _Otherwise. |
| 99 | template<typename _Cat, typename _Limit, typename _Otherwise = _Cat> |
| 100 | using __clamp_iter_cat |
| 101 | = conditional_t<derived_from<_Cat, _Limit>, _Limit, _Otherwise>; |
| 102 | } |
| 103 | #endif |
| 104 | |
| 105 | // 24.4.1 Reverse iterators |
| 106 | /** |
| 107 | * Bidirectional and random access iterators have corresponding reverse |
| 108 | * %iterator adaptors that iterate through the data structure in the |
| 109 | * opposite direction. They have the same signatures as the corresponding |
| 110 | * iterators. The fundamental relation between a reverse %iterator and its |
| 111 | * corresponding %iterator @c i is established by the identity: |
| 112 | * @code |
| 113 | * &*(reverse_iterator(i)) == &*(i - 1) |
| 114 | * @endcode |
| 115 | * |
| 116 | * <em>This mapping is dictated by the fact that while there is always a |
| 117 | * pointer past the end of an array, there might not be a valid pointer |
| 118 | * before the beginning of an array.</em> [24.4.1]/1,2 |
| 119 | * |
| 120 | * Reverse iterators can be tricky and surprising at first. Their |
| 121 | * semantics make sense, however, and the trickiness is a side effect of |
| 122 | * the requirement that the iterators must be safe. |
| 123 | */ |
| 124 | template<typename _Iterator> |
| 125 | class reverse_iterator |
| 126 | : public iterator<typename iterator_traits<_Iterator>::iterator_category, |
| 127 | typename iterator_traits<_Iterator>::value_type, |
| 128 | typename iterator_traits<_Iterator>::difference_type, |
| 129 | typename iterator_traits<_Iterator>::pointer, |
| 130 | typename iterator_traits<_Iterator>::reference> |
| 131 | { |
| 132 | protected: |
| 133 | _Iterator current; |
| 134 | |
| 135 | typedef iterator_traits<_Iterator> __traits_type; |
| 136 | |
| 137 | public: |
| 138 | typedef _Iterator iterator_type; |
| 139 | typedef typename __traits_type::difference_type difference_type; |
| 140 | typedef typename __traits_type::pointer pointer; |
| 141 | typedef typename __traits_type::reference reference; |
| 142 | |
| 143 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 144 | using iterator_concept |
| 145 | = conditional_t<random_access_iterator<_Iterator>, |
| 146 | random_access_iterator_tag, |
| 147 | bidirectional_iterator_tag>; |
| 148 | using iterator_category |
| 149 | = __detail::__clamp_iter_cat<typename __traits_type::iterator_category, |
| 150 | random_access_iterator_tag>; |
| 151 | #endif |
| 152 | |
| 153 | /** |
| 154 | * The default constructor value-initializes member @p current. |
| 155 | * If it is a pointer, that means it is zero-initialized. |
| 156 | */ |
| 157 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 158 | // 235 No specification of default ctor for reverse_iterator |
| 159 | // 1012. reverse_iterator default ctor should value initialize |
| 160 | _GLIBCXX17_CONSTEXPR |
| 161 | reverse_iterator() : current() { } |
| 162 | |
| 163 | /** |
| 164 | * This %iterator will move in the opposite direction that @p x does. |
| 165 | */ |
| 166 | explicit _GLIBCXX17_CONSTEXPR |
| 167 | reverse_iterator(iterator_type __x) : current(__x) { } |
| 168 | |
| 169 | /** |
| 170 | * The copy constructor is normal. |
| 171 | */ |
| 172 | _GLIBCXX17_CONSTEXPR |
| 173 | reverse_iterator(const reverse_iterator& __x) |
| 174 | : current(__x.current) { } |
| 175 | |
| 176 | #if __cplusplus201402L >= 201103L |
| 177 | reverse_iterator& operator=(const reverse_iterator&) = default; |
| 178 | #endif |
| 179 | |
| 180 | /** |
| 181 | * A %reverse_iterator across other types can be copied if the |
| 182 | * underlying %iterator can be converted to the type of @c current. |
| 183 | */ |
| 184 | template<typename _Iter> |
| 185 | _GLIBCXX17_CONSTEXPR |
| 186 | reverse_iterator(const reverse_iterator<_Iter>& __x) |
| 187 | : current(__x.base()) { } |
| 188 | |
| 189 | /** |
| 190 | * @return @c current, the %iterator used for underlying work. |
| 191 | */ |
| 192 | _GLIBCXX17_CONSTEXPR iterator_type |
| 193 | base() const |
| 194 | { return current; } |
| 195 | |
| 196 | /** |
| 197 | * @return A reference to the value at @c --current |
| 198 | * |
| 199 | * This requires that @c --current is dereferenceable. |
| 200 | * |
| 201 | * @warning This implementation requires that for an iterator of the |
| 202 | * underlying iterator type, @c x, a reference obtained by |
| 203 | * @c *x remains valid after @c x has been modified or |
| 204 | * destroyed. This is a bug: http://gcc.gnu.org/PR51823 |
| 205 | */ |
| 206 | _GLIBCXX17_CONSTEXPR reference |
| 207 | operator*() const |
| 208 | { |
| 209 | _Iterator __tmp = current; |
| 210 | return *--__tmp; |
| 211 | } |
| 212 | |
| 213 | /** |
| 214 | * @return A pointer to the value at @c --current |
| 215 | * |
| 216 | * This requires that @c --current is dereferenceable. |
| 217 | */ |
| 218 | _GLIBCXX17_CONSTEXPR pointer |
| 219 | operator->() const |
| 220 | #if __cplusplus201402L > 201703L && __cpp_concepts >= 201907L |
| 221 | requires is_pointer_v<_Iterator> |
| 222 | || requires(const _Iterator __i) { __i.operator->(); } |
| 223 | #endif |
| 224 | { |
| 225 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 226 | // 1052. operator-> should also support smart pointers |
| 227 | _Iterator __tmp = current; |
| 228 | --__tmp; |
| 229 | return _S_to_pointer(__tmp); |
| 230 | } |
| 231 | |
| 232 | /** |
| 233 | * @return @c *this |
| 234 | * |
| 235 | * Decrements the underlying iterator. |
| 236 | */ |
| 237 | _GLIBCXX17_CONSTEXPR reverse_iterator& |
| 238 | operator++() |
| 239 | { |
| 240 | --current; |
| 241 | return *this; |
| 242 | } |
| 243 | |
| 244 | /** |
| 245 | * @return The original value of @c *this |
| 246 | * |
| 247 | * Decrements the underlying iterator. |
| 248 | */ |
| 249 | _GLIBCXX17_CONSTEXPR reverse_iterator |
| 250 | operator++(int) |
| 251 | { |
| 252 | reverse_iterator __tmp = *this; |
| 253 | --current; |
| 254 | return __tmp; |
| 255 | } |
| 256 | |
| 257 | /** |
| 258 | * @return @c *this |
| 259 | * |
| 260 | * Increments the underlying iterator. |
| 261 | */ |
| 262 | _GLIBCXX17_CONSTEXPR reverse_iterator& |
| 263 | operator--() |
| 264 | { |
| 265 | ++current; |
| 266 | return *this; |
| 267 | } |
| 268 | |
| 269 | /** |
| 270 | * @return A reverse_iterator with the previous value of @c *this |
| 271 | * |
| 272 | * Increments the underlying iterator. |
| 273 | */ |
| 274 | _GLIBCXX17_CONSTEXPR reverse_iterator |
| 275 | operator--(int) |
| 276 | { |
| 277 | reverse_iterator __tmp = *this; |
| 278 | ++current; |
| 279 | return __tmp; |
| 280 | } |
| 281 | |
| 282 | /** |
| 283 | * @return A reverse_iterator that refers to @c current - @a __n |
| 284 | * |
| 285 | * The underlying iterator must be a Random Access Iterator. |
| 286 | */ |
| 287 | _GLIBCXX17_CONSTEXPR reverse_iterator |
| 288 | operator+(difference_type __n) const |
| 289 | { return reverse_iterator(current - __n); } |
| 290 | |
| 291 | /** |
| 292 | * @return *this |
| 293 | * |
| 294 | * Moves the underlying iterator backwards @a __n steps. |
| 295 | * The underlying iterator must be a Random Access Iterator. |
| 296 | */ |
| 297 | _GLIBCXX17_CONSTEXPR reverse_iterator& |
| 298 | operator+=(difference_type __n) |
| 299 | { |
| 300 | current -= __n; |
| 301 | return *this; |
| 302 | } |
| 303 | |
| 304 | /** |
| 305 | * @return A reverse_iterator that refers to @c current - @a __n |
| 306 | * |
| 307 | * The underlying iterator must be a Random Access Iterator. |
| 308 | */ |
| 309 | _GLIBCXX17_CONSTEXPR reverse_iterator |
| 310 | operator-(difference_type __n) const |
| 311 | { return reverse_iterator(current + __n); } |
| 312 | |
| 313 | /** |
| 314 | * @return *this |
| 315 | * |
| 316 | * Moves the underlying iterator forwards @a __n steps. |
| 317 | * The underlying iterator must be a Random Access Iterator. |
| 318 | */ |
| 319 | _GLIBCXX17_CONSTEXPR reverse_iterator& |
| 320 | operator-=(difference_type __n) |
| 321 | { |
| 322 | current += __n; |
| 323 | return *this; |
| 324 | } |
| 325 | |
| 326 | /** |
| 327 | * @return The value at @c current - @a __n - 1 |
| 328 | * |
| 329 | * The underlying iterator must be a Random Access Iterator. |
| 330 | */ |
| 331 | _GLIBCXX17_CONSTEXPR reference |
| 332 | operator[](difference_type __n) const |
| 333 | { return *(*this + __n); } |
| 334 | |
| 335 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 336 | friend constexpr iter_rvalue_reference_t<_Iterator> |
| 337 | iter_move(const reverse_iterator& __i) |
| 338 | noexcept(is_nothrow_copy_constructible_v<_Iterator> |
| 339 | && noexcept(ranges::iter_move(--std::declval<_Iterator&>()))) |
| 340 | { |
| 341 | auto __tmp = __i.base(); |
| 342 | return ranges::iter_move(--__tmp); |
| 343 | } |
| 344 | |
| 345 | template<indirectly_swappable<_Iterator> _Iter2> |
| 346 | friend constexpr void |
| 347 | iter_swap(const reverse_iterator& __x, |
| 348 | const reverse_iterator<_Iter2>& __y) |
| 349 | noexcept(is_nothrow_copy_constructible_v<_Iterator> |
| 350 | && is_nothrow_copy_constructible_v<_Iter2> |
| 351 | && noexcept(ranges::iter_swap(--std::declval<_Iterator&>(), |
| 352 | --std::declval<_Iter2&>()))) |
| 353 | { |
| 354 | auto __xtmp = __x.base(); |
| 355 | auto __ytmp = __y.base(); |
| 356 | ranges::iter_swap(--__xtmp, --__ytmp); |
| 357 | } |
| 358 | #endif |
| 359 | |
| 360 | private: |
| 361 | template<typename _Tp> |
| 362 | static _GLIBCXX17_CONSTEXPR _Tp* |
| 363 | _S_to_pointer(_Tp* __p) |
| 364 | { return __p; } |
| 365 | |
| 366 | template<typename _Tp> |
| 367 | static _GLIBCXX17_CONSTEXPR pointer |
| 368 | _S_to_pointer(_Tp __t) |
| 369 | { return __t.operator->(); } |
| 370 | }; |
| 371 | |
| 372 | //@{ |
| 373 | /** |
| 374 | * @param __x A %reverse_iterator. |
| 375 | * @param __y A %reverse_iterator. |
| 376 | * @return A simple bool. |
| 377 | * |
| 378 | * Reverse iterators forward comparisons to their underlying base() |
| 379 | * iterators. |
| 380 | * |
| 381 | */ |
| 382 | #if __cplusplus201402L <= 201703L || ! defined __cpp_lib_concepts |
| 383 | template<typename _Iterator> |
| 384 | inline _GLIBCXX17_CONSTEXPR bool |
| 385 | operator==(const reverse_iterator<_Iterator>& __x, |
| 386 | const reverse_iterator<_Iterator>& __y) |
| 387 | { return __x.base() == __y.base(); } |
| 388 | |
| 389 | template<typename _Iterator> |
| 390 | inline _GLIBCXX17_CONSTEXPR bool |
| 391 | operator<(const reverse_iterator<_Iterator>& __x, |
| 392 | const reverse_iterator<_Iterator>& __y) |
| 393 | { return __y.base() < __x.base(); } |
| 394 | |
| 395 | template<typename _Iterator> |
| 396 | inline _GLIBCXX17_CONSTEXPR bool |
| 397 | operator!=(const reverse_iterator<_Iterator>& __x, |
| 398 | const reverse_iterator<_Iterator>& __y) |
| 399 | { return !(__x == __y); } |
| 400 | |
| 401 | template<typename _Iterator> |
| 402 | inline _GLIBCXX17_CONSTEXPR bool |
| 403 | operator>(const reverse_iterator<_Iterator>& __x, |
| 404 | const reverse_iterator<_Iterator>& __y) |
| 405 | { return __y < __x; } |
| 406 | |
| 407 | template<typename _Iterator> |
| 408 | inline _GLIBCXX17_CONSTEXPR bool |
| 409 | operator<=(const reverse_iterator<_Iterator>& __x, |
| 410 | const reverse_iterator<_Iterator>& __y) |
| 411 | { return !(__y < __x); } |
| 412 | |
| 413 | template<typename _Iterator> |
| 414 | inline _GLIBCXX17_CONSTEXPR bool |
| 415 | operator>=(const reverse_iterator<_Iterator>& __x, |
| 416 | const reverse_iterator<_Iterator>& __y) |
| 417 | { return !(__x < __y); } |
| 418 | |
| 419 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 420 | // DR 280. Comparison of reverse_iterator to const reverse_iterator. |
| 421 | template<typename _IteratorL, typename _IteratorR> |
| 422 | inline _GLIBCXX17_CONSTEXPR bool |
| 423 | operator==(const reverse_iterator<_IteratorL>& __x, |
| 424 | const reverse_iterator<_IteratorR>& __y) |
| 425 | { return __x.base() == __y.base(); } |
| 426 | |
| 427 | template<typename _IteratorL, typename _IteratorR> |
| 428 | inline _GLIBCXX17_CONSTEXPR bool |
| 429 | operator<(const reverse_iterator<_IteratorL>& __x, |
| 430 | const reverse_iterator<_IteratorR>& __y) |
| 431 | { return __y.base() < __x.base(); } |
| 432 | |
| 433 | template<typename _IteratorL, typename _IteratorR> |
| 434 | inline _GLIBCXX17_CONSTEXPR bool |
| 435 | operator!=(const reverse_iterator<_IteratorL>& __x, |
| 436 | const reverse_iterator<_IteratorR>& __y) |
| 437 | { return !(__x == __y); } |
| 438 | |
| 439 | template<typename _IteratorL, typename _IteratorR> |
| 440 | inline _GLIBCXX17_CONSTEXPR bool |
| 441 | operator>(const reverse_iterator<_IteratorL>& __x, |
| 442 | const reverse_iterator<_IteratorR>& __y) |
| 443 | { return __y < __x; } |
| 444 | |
| 445 | template<typename _IteratorL, typename _IteratorR> |
| 446 | inline _GLIBCXX17_CONSTEXPR bool |
| 447 | operator<=(const reverse_iterator<_IteratorL>& __x, |
| 448 | const reverse_iterator<_IteratorR>& __y) |
| 449 | { return !(__y < __x); } |
| 450 | |
| 451 | template<typename _IteratorL, typename _IteratorR> |
| 452 | inline _GLIBCXX17_CONSTEXPR bool |
| 453 | operator>=(const reverse_iterator<_IteratorL>& __x, |
| 454 | const reverse_iterator<_IteratorR>& __y) |
| 455 | { return !(__x < __y); } |
| 456 | #else // C++20 |
| 457 | template<typename _IteratorL, typename _IteratorR> |
| 458 | constexpr bool |
| 459 | operator==(const reverse_iterator<_IteratorL>& __x, |
| 460 | const reverse_iterator<_IteratorR>& __y) |
| 461 | requires requires { { __x.base() == __y.base() } -> convertible_to<bool>; } |
| 462 | { return __x.base() == __y.base(); } |
| 463 | |
| 464 | template<typename _IteratorL, typename _IteratorR> |
| 465 | constexpr bool |
| 466 | operator!=(const reverse_iterator<_IteratorL>& __x, |
| 467 | const reverse_iterator<_IteratorR>& __y) |
| 468 | requires requires { { __x.base() != __y.base() } -> convertible_to<bool>; } |
| 469 | { return __x.base() != __y.base(); } |
| 470 | |
| 471 | template<typename _IteratorL, typename _IteratorR> |
| 472 | constexpr bool |
| 473 | operator<(const reverse_iterator<_IteratorL>& __x, |
| 474 | const reverse_iterator<_IteratorR>& __y) |
| 475 | requires requires { { __x.base() > __y.base() } -> convertible_to<bool>; } |
| 476 | { return __x.base() > __y.base(); } |
| 477 | |
| 478 | template<typename _IteratorL, typename _IteratorR> |
| 479 | constexpr bool |
| 480 | operator>(const reverse_iterator<_IteratorL>& __x, |
| 481 | const reverse_iterator<_IteratorR>& __y) |
| 482 | requires requires { { __x.base() < __y.base() } -> convertible_to<bool>; } |
| 483 | { return __x.base() < __y.base(); } |
| 484 | |
| 485 | template<typename _IteratorL, typename _IteratorR> |
| 486 | constexpr bool |
| 487 | operator<=(const reverse_iterator<_IteratorL>& __x, |
| 488 | const reverse_iterator<_IteratorR>& __y) |
| 489 | requires requires { { __x.base() >= __y.base() } -> convertible_to<bool>; } |
| 490 | { return __x.base() >= __y.base(); } |
| 491 | |
| 492 | template<typename _IteratorL, typename _IteratorR> |
| 493 | constexpr bool |
| 494 | operator>=(const reverse_iterator<_IteratorL>& __x, |
| 495 | const reverse_iterator<_IteratorR>& __y) |
| 496 | requires requires { { __x.base() <= __y.base() } -> convertible_to<bool>; } |
| 497 | { return __x.base() <= __y.base(); } |
| 498 | |
| 499 | template<typename _IteratorL, |
| 500 | three_way_comparable_with<_IteratorL> _IteratorR> |
| 501 | constexpr compare_three_way_result_t<_IteratorL, _IteratorR> |
| 502 | operator<=>(const reverse_iterator<_IteratorL>& __x, |
| 503 | const reverse_iterator<_IteratorR>& __y) |
| 504 | { return __y.base() <=> __x.base(); } |
| 505 | #endif // C++20 |
| 506 | //@} |
| 507 | |
| 508 | #if __cplusplus201402L < 201103L |
| 509 | template<typename _Iterator> |
| 510 | inline typename reverse_iterator<_Iterator>::difference_type |
| 511 | operator-(const reverse_iterator<_Iterator>& __x, |
| 512 | const reverse_iterator<_Iterator>& __y) |
| 513 | { return __y.base() - __x.base(); } |
| 514 | |
| 515 | template<typename _IteratorL, typename _IteratorR> |
| 516 | inline typename reverse_iterator<_IteratorL>::difference_type |
| 517 | operator-(const reverse_iterator<_IteratorL>& __x, |
| 518 | const reverse_iterator<_IteratorR>& __y) |
| 519 | { return __y.base() - __x.base(); } |
| 520 | #else |
| 521 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 522 | // DR 685. reverse_iterator/move_iterator difference has invalid signatures |
| 523 | template<typename _IteratorL, typename _IteratorR> |
| 524 | inline _GLIBCXX17_CONSTEXPR auto |
| 525 | operator-(const reverse_iterator<_IteratorL>& __x, |
| 526 | const reverse_iterator<_IteratorR>& __y) |
| 527 | -> decltype(__y.base() - __x.base()) |
| 528 | { return __y.base() - __x.base(); } |
| 529 | #endif |
| 530 | |
| 531 | template<typename _Iterator> |
| 532 | inline _GLIBCXX17_CONSTEXPR reverse_iterator<_Iterator> |
| 533 | operator+(typename reverse_iterator<_Iterator>::difference_type __n, |
| 534 | const reverse_iterator<_Iterator>& __x) |
| 535 | { return reverse_iterator<_Iterator>(__x.base() - __n); } |
| 536 | |
| 537 | #if __cplusplus201402L >= 201103L |
| 538 | // Same as C++14 make_reverse_iterator but used in C++11 mode too. |
| 539 | template<typename _Iterator> |
| 540 | inline _GLIBCXX17_CONSTEXPR reverse_iterator<_Iterator> |
| 541 | __make_reverse_iterator(_Iterator __i) |
| 542 | { return reverse_iterator<_Iterator>(__i); } |
| 543 | |
| 544 | # if __cplusplus201402L >= 201402L |
| 545 | # define __cpp_lib_make_reverse_iterator201402 201402 |
| 546 | |
| 547 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 548 | // DR 2285. make_reverse_iterator |
| 549 | /// Generator function for reverse_iterator. |
| 550 | template<typename _Iterator> |
| 551 | inline _GLIBCXX17_CONSTEXPR reverse_iterator<_Iterator> |
| 552 | make_reverse_iterator(_Iterator __i) |
| 553 | { return reverse_iterator<_Iterator>(__i); } |
| 554 | |
| 555 | # if __cplusplus201402L > 201703L && defined __cpp_lib_concepts |
| 556 | template<typename _Iterator1, typename _Iterator2> |
| 557 | requires (!sized_sentinel_for<_Iterator1, _Iterator2>) |
| 558 | inline constexpr bool |
| 559 | disable_sized_sentinel_for<reverse_iterator<_Iterator1>, |
| 560 | reverse_iterator<_Iterator2>> = true; |
| 561 | # endif // C++20 |
| 562 | # endif // C++14 |
| 563 | |
| 564 | template<typename _Iterator> |
| 565 | _GLIBCXX20_CONSTEXPR |
| 566 | auto |
| 567 | __niter_base(reverse_iterator<_Iterator> __it) |
| 568 | -> decltype(__make_reverse_iterator(__niter_base(__it.base()))) |
| 569 | { return __make_reverse_iterator(__niter_base(__it.base())); } |
| 570 | |
| 571 | template<typename _Iterator> |
| 572 | struct __is_move_iterator<reverse_iterator<_Iterator> > |
| 573 | : __is_move_iterator<_Iterator> |
| 574 | { }; |
| 575 | |
| 576 | template<typename _Iterator> |
| 577 | _GLIBCXX20_CONSTEXPR |
| 578 | auto |
| 579 | __miter_base(reverse_iterator<_Iterator> __it) |
| 580 | -> decltype(__make_reverse_iterator(__miter_base(__it.base()))) |
| 581 | { return __make_reverse_iterator(__miter_base(__it.base())); } |
| 582 | #endif // C++11 |
| 583 | |
| 584 | // 24.4.2.2.1 back_insert_iterator |
| 585 | /** |
| 586 | * @brief Turns assignment into insertion. |
| 587 | * |
| 588 | * These are output iterators, constructed from a container-of-T. |
| 589 | * Assigning a T to the iterator appends it to the container using |
| 590 | * push_back. |
| 591 | * |
| 592 | * Tip: Using the back_inserter function to create these iterators can |
| 593 | * save typing. |
| 594 | */ |
| 595 | template<typename _Container> |
| 596 | class back_insert_iterator |
| 597 | : public iterator<output_iterator_tag, void, void, void, void> |
| 598 | { |
| 599 | protected: |
| 600 | _Container* container; |
| 601 | |
| 602 | public: |
| 603 | /// A nested typedef for the type of whatever container you used. |
| 604 | typedef _Container container_type; |
| 605 | #if __cplusplus201402L > 201703L |
| 606 | using difference_type = ptrdiff_t; |
| 607 | |
| 608 | constexpr back_insert_iterator() noexcept : container(nullptr) { } |
| 609 | #endif |
| 610 | |
| 611 | /// The only way to create this %iterator is with a container. |
| 612 | explicit _GLIBCXX20_CONSTEXPR |
| 613 | back_insert_iterator(_Container& __x) |
| 614 | : container(std::__addressof(__x)) { } |
| 615 | |
| 616 | /** |
| 617 | * @param __value An instance of whatever type |
| 618 | * container_type::const_reference is; presumably a |
| 619 | * reference-to-const T for container<T>. |
| 620 | * @return This %iterator, for chained operations. |
| 621 | * |
| 622 | * This kind of %iterator doesn't really have a @a position in the |
| 623 | * container (you can think of the position as being permanently at |
| 624 | * the end, if you like). Assigning a value to the %iterator will |
| 625 | * always append the value to the end of the container. |
| 626 | */ |
| 627 | #if __cplusplus201402L < 201103L |
| 628 | back_insert_iterator& |
| 629 | operator=(typename _Container::const_reference __value) |
| 630 | { |
| 631 | container->push_back(__value); |
| 632 | return *this; |
| 633 | } |
| 634 | #else |
| 635 | _GLIBCXX20_CONSTEXPR |
| 636 | back_insert_iterator& |
| 637 | operator=(const typename _Container::value_type& __value) |
| 638 | { |
| 639 | container->push_back(__value); |
| 640 | return *this; |
| 641 | } |
| 642 | |
| 643 | _GLIBCXX20_CONSTEXPR |
| 644 | back_insert_iterator& |
| 645 | operator=(typename _Container::value_type&& __value) |
| 646 | { |
| 647 | container->push_back(std::move(__value)); |
| 648 | return *this; |
| 649 | } |
| 650 | #endif |
| 651 | |
| 652 | /// Simply returns *this. |
| 653 | _GLIBCXX20_CONSTEXPR |
| 654 | back_insert_iterator& |
| 655 | operator*() |
| 656 | { return *this; } |
| 657 | |
| 658 | /// Simply returns *this. (This %iterator does not @a move.) |
| 659 | _GLIBCXX20_CONSTEXPR |
| 660 | back_insert_iterator& |
| 661 | operator++() |
| 662 | { return *this; } |
| 663 | |
| 664 | /// Simply returns *this. (This %iterator does not @a move.) |
| 665 | _GLIBCXX20_CONSTEXPR |
| 666 | back_insert_iterator |
| 667 | operator++(int) |
| 668 | { return *this; } |
| 669 | }; |
| 670 | |
| 671 | /** |
| 672 | * @param __x A container of arbitrary type. |
| 673 | * @return An instance of back_insert_iterator working on @p __x. |
| 674 | * |
| 675 | * This wrapper function helps in creating back_insert_iterator instances. |
| 676 | * Typing the name of the %iterator requires knowing the precise full |
| 677 | * type of the container, which can be tedious and impedes generic |
| 678 | * programming. Using this function lets you take advantage of automatic |
| 679 | * template parameter deduction, making the compiler match the correct |
| 680 | * types for you. |
| 681 | */ |
| 682 | template<typename _Container> |
| 683 | _GLIBCXX20_CONSTEXPR |
| 684 | inline back_insert_iterator<_Container> |
| 685 | back_inserter(_Container& __x) |
| 686 | { return back_insert_iterator<_Container>(__x); } |
| 687 | |
| 688 | /** |
| 689 | * @brief Turns assignment into insertion. |
| 690 | * |
| 691 | * These are output iterators, constructed from a container-of-T. |
| 692 | * Assigning a T to the iterator prepends it to the container using |
| 693 | * push_front. |
| 694 | * |
| 695 | * Tip: Using the front_inserter function to create these iterators can |
| 696 | * save typing. |
| 697 | */ |
| 698 | template<typename _Container> |
| 699 | class front_insert_iterator |
| 700 | : public iterator<output_iterator_tag, void, void, void, void> |
| 701 | { |
| 702 | protected: |
| 703 | _Container* container; |
| 704 | |
| 705 | public: |
| 706 | /// A nested typedef for the type of whatever container you used. |
| 707 | typedef _Container container_type; |
| 708 | #if __cplusplus201402L > 201703L |
| 709 | using difference_type = ptrdiff_t; |
| 710 | |
| 711 | constexpr front_insert_iterator() noexcept : container(nullptr) { } |
| 712 | #endif |
| 713 | |
| 714 | /// The only way to create this %iterator is with a container. |
| 715 | explicit _GLIBCXX20_CONSTEXPR |
| 716 | front_insert_iterator(_Container& __x) |
| 717 | : container(std::__addressof(__x)) { } |
| 718 | |
| 719 | /** |
| 720 | * @param __value An instance of whatever type |
| 721 | * container_type::const_reference is; presumably a |
| 722 | * reference-to-const T for container<T>. |
| 723 | * @return This %iterator, for chained operations. |
| 724 | * |
| 725 | * This kind of %iterator doesn't really have a @a position in the |
| 726 | * container (you can think of the position as being permanently at |
| 727 | * the front, if you like). Assigning a value to the %iterator will |
| 728 | * always prepend the value to the front of the container. |
| 729 | */ |
| 730 | #if __cplusplus201402L < 201103L |
| 731 | front_insert_iterator& |
| 732 | operator=(typename _Container::const_reference __value) |
| 733 | { |
| 734 | container->push_front(__value); |
| 735 | return *this; |
| 736 | } |
| 737 | #else |
| 738 | _GLIBCXX20_CONSTEXPR |
| 739 | front_insert_iterator& |
| 740 | operator=(const typename _Container::value_type& __value) |
| 741 | { |
| 742 | container->push_front(__value); |
| 743 | return *this; |
| 744 | } |
| 745 | |
| 746 | _GLIBCXX20_CONSTEXPR |
| 747 | front_insert_iterator& |
| 748 | operator=(typename _Container::value_type&& __value) |
| 749 | { |
| 750 | container->push_front(std::move(__value)); |
| 751 | return *this; |
| 752 | } |
| 753 | #endif |
| 754 | |
| 755 | /// Simply returns *this. |
| 756 | _GLIBCXX20_CONSTEXPR |
| 757 | front_insert_iterator& |
| 758 | operator*() |
| 759 | { return *this; } |
| 760 | |
| 761 | /// Simply returns *this. (This %iterator does not @a move.) |
| 762 | _GLIBCXX20_CONSTEXPR |
| 763 | front_insert_iterator& |
| 764 | operator++() |
| 765 | { return *this; } |
| 766 | |
| 767 | /// Simply returns *this. (This %iterator does not @a move.) |
| 768 | _GLIBCXX20_CONSTEXPR |
| 769 | front_insert_iterator |
| 770 | operator++(int) |
| 771 | { return *this; } |
| 772 | }; |
| 773 | |
| 774 | /** |
| 775 | * @param __x A container of arbitrary type. |
| 776 | * @return An instance of front_insert_iterator working on @p x. |
| 777 | * |
| 778 | * This wrapper function helps in creating front_insert_iterator instances. |
| 779 | * Typing the name of the %iterator requires knowing the precise full |
| 780 | * type of the container, which can be tedious and impedes generic |
| 781 | * programming. Using this function lets you take advantage of automatic |
| 782 | * template parameter deduction, making the compiler match the correct |
| 783 | * types for you. |
| 784 | */ |
| 785 | template<typename _Container> |
| 786 | _GLIBCXX20_CONSTEXPR |
| 787 | inline front_insert_iterator<_Container> |
| 788 | front_inserter(_Container& __x) |
| 789 | { return front_insert_iterator<_Container>(__x); } |
| 790 | |
| 791 | /** |
| 792 | * @brief Turns assignment into insertion. |
| 793 | * |
| 794 | * These are output iterators, constructed from a container-of-T. |
| 795 | * Assigning a T to the iterator inserts it in the container at the |
| 796 | * %iterator's position, rather than overwriting the value at that |
| 797 | * position. |
| 798 | * |
| 799 | * (Sequences will actually insert a @e copy of the value before the |
| 800 | * %iterator's position.) |
| 801 | * |
| 802 | * Tip: Using the inserter function to create these iterators can |
| 803 | * save typing. |
| 804 | */ |
| 805 | template<typename _Container> |
| 806 | class insert_iterator |
| 807 | : public iterator<output_iterator_tag, void, void, void, void> |
| 808 | { |
| 809 | #if __cplusplus201402L > 201703L && defined __cpp_lib_concepts |
| 810 | using _Iter = std::__detail::__range_iter_t<_Container>; |
| 811 | |
| 812 | protected: |
| 813 | _Container* container = nullptr; |
| 814 | _Iter iter = _Iter(); |
| 815 | #else |
| 816 | typedef typename _Container::iterator _Iter; |
| 817 | |
| 818 | protected: |
| 819 | _Container* container; |
| 820 | _Iter iter; |
| 821 | #endif |
| 822 | |
| 823 | public: |
| 824 | /// A nested typedef for the type of whatever container you used. |
| 825 | typedef _Container container_type; |
| 826 | |
| 827 | #if __cplusplus201402L > 201703L && defined __cpp_lib_concepts |
| 828 | using difference_type = ptrdiff_t; |
| 829 | |
| 830 | insert_iterator() = default; |
| 831 | #endif |
| 832 | |
| 833 | /** |
| 834 | * The only way to create this %iterator is with a container and an |
| 835 | * initial position (a normal %iterator into the container). |
| 836 | */ |
| 837 | _GLIBCXX20_CONSTEXPR |
| 838 | insert_iterator(_Container& __x, _Iter __i) |
| 839 | : container(std::__addressof(__x)), iter(__i) {} |
| 840 | |
| 841 | /** |
| 842 | * @param __value An instance of whatever type |
| 843 | * container_type::const_reference is; presumably a |
| 844 | * reference-to-const T for container<T>. |
| 845 | * @return This %iterator, for chained operations. |
| 846 | * |
| 847 | * This kind of %iterator maintains its own position in the |
| 848 | * container. Assigning a value to the %iterator will insert the |
| 849 | * value into the container at the place before the %iterator. |
| 850 | * |
| 851 | * The position is maintained such that subsequent assignments will |
| 852 | * insert values immediately after one another. For example, |
| 853 | * @code |
| 854 | * // vector v contains A and Z |
| 855 | * |
| 856 | * insert_iterator i (v, ++v.begin()); |
| 857 | * i = 1; |
| 858 | * i = 2; |
| 859 | * i = 3; |
| 860 | * |
| 861 | * // vector v contains A, 1, 2, 3, and Z |
| 862 | * @endcode |
| 863 | */ |
| 864 | #if __cplusplus201402L < 201103L |
| 865 | insert_iterator& |
| 866 | operator=(typename _Container::const_reference __value) |
| 867 | { |
| 868 | iter = container->insert(iter, __value); |
| 869 | ++iter; |
| 870 | return *this; |
| 871 | } |
| 872 | #else |
| 873 | _GLIBCXX20_CONSTEXPR |
| 874 | insert_iterator& |
| 875 | operator=(const typename _Container::value_type& __value) |
| 876 | { |
| 877 | iter = container->insert(iter, __value); |
| 878 | ++iter; |
| 879 | return *this; |
| 880 | } |
| 881 | |
| 882 | _GLIBCXX20_CONSTEXPR |
| 883 | insert_iterator& |
| 884 | operator=(typename _Container::value_type&& __value) |
| 885 | { |
| 886 | iter = container->insert(iter, std::move(__value)); |
| 887 | ++iter; |
| 888 | return *this; |
| 889 | } |
| 890 | #endif |
| 891 | |
| 892 | /// Simply returns *this. |
| 893 | _GLIBCXX20_CONSTEXPR |
| 894 | insert_iterator& |
| 895 | operator*() |
| 896 | { return *this; } |
| 897 | |
| 898 | /// Simply returns *this. (This %iterator does not @a move.) |
| 899 | _GLIBCXX20_CONSTEXPR |
| 900 | insert_iterator& |
| 901 | operator++() |
| 902 | { return *this; } |
| 903 | |
| 904 | /// Simply returns *this. (This %iterator does not @a move.) |
| 905 | _GLIBCXX20_CONSTEXPR |
| 906 | insert_iterator& |
| 907 | operator++(int) |
| 908 | { return *this; } |
| 909 | }; |
| 910 | |
| 911 | /** |
| 912 | * @param __x A container of arbitrary type. |
| 913 | * @param __i An iterator into the container. |
| 914 | * @return An instance of insert_iterator working on @p __x. |
| 915 | * |
| 916 | * This wrapper function helps in creating insert_iterator instances. |
| 917 | * Typing the name of the %iterator requires knowing the precise full |
| 918 | * type of the container, which can be tedious and impedes generic |
| 919 | * programming. Using this function lets you take advantage of automatic |
| 920 | * template parameter deduction, making the compiler match the correct |
| 921 | * types for you. |
| 922 | */ |
| 923 | #if __cplusplus201402L > 201703L && defined __cpp_lib_concepts |
| 924 | template<typename _Container> |
| 925 | constexpr insert_iterator<_Container> |
| 926 | inserter(_Container& __x, std::__detail::__range_iter_t<_Container> __i) |
| 927 | { return insert_iterator<_Container>(__x, __i); } |
| 928 | #else |
| 929 | template<typename _Container, typename _Iterator> |
| 930 | inline insert_iterator<_Container> |
| 931 | inserter(_Container& __x, _Iterator __i) |
| 932 | { |
| 933 | return insert_iterator<_Container>(__x, |
| 934 | typename _Container::iterator(__i)); |
| 935 | } |
| 936 | #endif |
| 937 | |
| 938 | // @} group iterators |
| 939 | |
| 940 | _GLIBCXX_END_NAMESPACE_VERSION |
| 941 | } // namespace |
| 942 | |
| 943 | namespace __gnu_cxx _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default"))) |
| 944 | { |
| 945 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
| 946 | |
| 947 | // This iterator adapter is @a normal in the sense that it does not |
| 948 | // change the semantics of any of the operators of its iterator |
| 949 | // parameter. Its primary purpose is to convert an iterator that is |
| 950 | // not a class, e.g. a pointer, into an iterator that is a class. |
| 951 | // The _Container parameter exists solely so that different containers |
| 952 | // using this template can instantiate different types, even if the |
| 953 | // _Iterator parameter is the same. |
| 954 | template<typename _Iterator, typename _Container> |
| 955 | class __normal_iterator |
| 956 | { |
| 957 | protected: |
| 958 | _Iterator _M_current; |
| 959 | |
| 960 | typedef std::iterator_traits<_Iterator> __traits_type; |
| 961 | |
| 962 | public: |
| 963 | typedef _Iterator iterator_type; |
| 964 | typedef typename __traits_type::iterator_category iterator_category; |
| 965 | typedef typename __traits_type::value_type value_type; |
| 966 | typedef typename __traits_type::difference_type difference_type; |
| 967 | typedef typename __traits_type::reference reference; |
| 968 | typedef typename __traits_type::pointer pointer; |
| 969 | |
| 970 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 971 | using iterator_concept = std::__detail::__iter_concept<_Iterator>; |
| 972 | #endif |
| 973 | |
| 974 | _GLIBCXX_CONSTEXPRconstexpr __normal_iterator() _GLIBCXX_NOEXCEPTnoexcept |
| 975 | : _M_current(_Iterator()) { } |
| 976 | |
| 977 | explicit _GLIBCXX20_CONSTEXPR |
| 978 | __normal_iterator(const _Iterator& __i) _GLIBCXX_NOEXCEPTnoexcept |
| 979 | : _M_current(__i) { } |
| 980 | |
| 981 | // Allow iterator to const_iterator conversion |
| 982 | template<typename _Iter> |
| 983 | _GLIBCXX20_CONSTEXPR |
| 984 | __normal_iterator(const __normal_iterator<_Iter, |
| 985 | typename __enable_if< |
| 986 | (std::__are_same<_Iter, typename _Container::pointer>::__value), |
| 987 | _Container>::__type>& __i) _GLIBCXX_NOEXCEPTnoexcept |
| 988 | : _M_current(__i.base()) { } |
| 989 | |
| 990 | // Forward iterator requirements |
| 991 | _GLIBCXX20_CONSTEXPR |
| 992 | reference |
| 993 | operator*() const _GLIBCXX_NOEXCEPTnoexcept |
| 994 | { return *_M_current; } |
| 995 | |
| 996 | _GLIBCXX20_CONSTEXPR |
| 997 | pointer |
| 998 | operator->() const _GLIBCXX_NOEXCEPTnoexcept |
| 999 | { return _M_current; } |
| 1000 | |
| 1001 | _GLIBCXX20_CONSTEXPR |
| 1002 | __normal_iterator& |
| 1003 | operator++() _GLIBCXX_NOEXCEPTnoexcept |
| 1004 | { |
| 1005 | ++_M_current; |
| 1006 | return *this; |
| 1007 | } |
| 1008 | |
| 1009 | _GLIBCXX20_CONSTEXPR |
| 1010 | __normal_iterator |
| 1011 | operator++(int) _GLIBCXX_NOEXCEPTnoexcept |
| 1012 | { return __normal_iterator(_M_current++); } |
| 1013 | |
| 1014 | // Bidirectional iterator requirements |
| 1015 | _GLIBCXX20_CONSTEXPR |
| 1016 | __normal_iterator& |
| 1017 | operator--() _GLIBCXX_NOEXCEPTnoexcept |
| 1018 | { |
| 1019 | --_M_current; |
| 1020 | return *this; |
| 1021 | } |
| 1022 | |
| 1023 | _GLIBCXX20_CONSTEXPR |
| 1024 | __normal_iterator |
| 1025 | operator--(int) _GLIBCXX_NOEXCEPTnoexcept |
| 1026 | { return __normal_iterator(_M_current--); } |
| 1027 | |
| 1028 | // Random access iterator requirements |
| 1029 | _GLIBCXX20_CONSTEXPR |
| 1030 | reference |
| 1031 | operator[](difference_type __n) const _GLIBCXX_NOEXCEPTnoexcept |
| 1032 | { return _M_current[__n]; } |
| 1033 | |
| 1034 | _GLIBCXX20_CONSTEXPR |
| 1035 | __normal_iterator& |
| 1036 | operator+=(difference_type __n) _GLIBCXX_NOEXCEPTnoexcept |
| 1037 | { _M_current += __n; return *this; } |
| 1038 | |
| 1039 | _GLIBCXX20_CONSTEXPR |
| 1040 | __normal_iterator |
| 1041 | operator+(difference_type __n) const _GLIBCXX_NOEXCEPTnoexcept |
| 1042 | { return __normal_iterator(_M_current + __n); } |
| 1043 | |
| 1044 | _GLIBCXX20_CONSTEXPR |
| 1045 | __normal_iterator& |
| 1046 | operator-=(difference_type __n) _GLIBCXX_NOEXCEPTnoexcept |
| 1047 | { _M_current -= __n; return *this; } |
| 1048 | |
| 1049 | _GLIBCXX20_CONSTEXPR |
| 1050 | __normal_iterator |
| 1051 | operator-(difference_type __n) const _GLIBCXX_NOEXCEPTnoexcept |
| 1052 | { return __normal_iterator(_M_current - __n); } |
| 1053 | |
| 1054 | _GLIBCXX20_CONSTEXPR |
| 1055 | const _Iterator& |
| 1056 | base() const _GLIBCXX_NOEXCEPTnoexcept |
| 1057 | { return _M_current; } |
| 1058 | }; |
| 1059 | |
| 1060 | // Note: In what follows, the left- and right-hand-side iterators are |
| 1061 | // allowed to vary in types (conceptually in cv-qualification) so that |
| 1062 | // comparison between cv-qualified and non-cv-qualified iterators be |
| 1063 | // valid. However, the greedy and unfriendly operators in std::rel_ops |
| 1064 | // will make overload resolution ambiguous (when in scope) if we don't |
| 1065 | // provide overloads whose operands are of the same type. Can someone |
| 1066 | // remind me what generic programming is about? -- Gaby |
| 1067 | |
| 1068 | #if __cpp_lib_three_way_comparison |
| 1069 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1070 | requires requires (_IteratorL __lhs, _IteratorR __rhs) |
| 1071 | { { __lhs == __rhs } -> std::convertible_to<bool>; } |
| 1072 | constexpr bool |
| 1073 | operator==(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1074 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1075 | noexcept(noexcept(__lhs.base() == __rhs.base())) |
| 1076 | { return __lhs.base() == __rhs.base(); } |
| 1077 | |
| 1078 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1079 | constexpr std::__detail::__synth3way_t<_IteratorR, _IteratorL> |
| 1080 | operator<=>(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1081 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1082 | noexcept(noexcept(std::__detail::__synth3way(__lhs.base(), __rhs.base()))) |
| 1083 | { return std::__detail::__synth3way(__lhs.base(), __rhs.base()); } |
| 1084 | #else |
| 1085 | // Forward iterator requirements |
| 1086 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1087 | _GLIBCXX20_CONSTEXPR |
| 1088 | inline bool |
| 1089 | operator==(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1090 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1091 | _GLIBCXX_NOEXCEPTnoexcept |
| 1092 | { return __lhs.base() == __rhs.base(); } |
| 1093 | |
| 1094 | template<typename _Iterator, typename _Container> |
| 1095 | _GLIBCXX20_CONSTEXPR |
| 1096 | inline bool |
| 1097 | operator==(const __normal_iterator<_Iterator, _Container>& __lhs, |
| 1098 | const __normal_iterator<_Iterator, _Container>& __rhs) |
| 1099 | _GLIBCXX_NOEXCEPTnoexcept |
| 1100 | { return __lhs.base() == __rhs.base(); } |
| 1101 | |
| 1102 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1103 | _GLIBCXX20_CONSTEXPR |
| 1104 | inline bool |
| 1105 | operator!=(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1106 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1107 | _GLIBCXX_NOEXCEPTnoexcept |
| 1108 | { return __lhs.base() != __rhs.base(); } |
| 1109 | |
| 1110 | template<typename _Iterator, typename _Container> |
| 1111 | _GLIBCXX20_CONSTEXPR |
| 1112 | inline bool |
| 1113 | operator!=(const __normal_iterator<_Iterator, _Container>& __lhs, |
| 1114 | const __normal_iterator<_Iterator, _Container>& __rhs) |
| 1115 | _GLIBCXX_NOEXCEPTnoexcept |
| 1116 | { return __lhs.base() != __rhs.base(); } |
| 1117 | |
| 1118 | // Random access iterator requirements |
| 1119 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1120 | inline bool |
| 1121 | operator<(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1122 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1123 | _GLIBCXX_NOEXCEPTnoexcept |
| 1124 | { return __lhs.base() < __rhs.base(); } |
| 1125 | |
| 1126 | template<typename _Iterator, typename _Container> |
| 1127 | _GLIBCXX20_CONSTEXPR |
| 1128 | inline bool |
| 1129 | operator<(const __normal_iterator<_Iterator, _Container>& __lhs, |
| 1130 | const __normal_iterator<_Iterator, _Container>& __rhs) |
| 1131 | _GLIBCXX_NOEXCEPTnoexcept |
| 1132 | { return __lhs.base() < __rhs.base(); } |
| 1133 | |
| 1134 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1135 | inline bool |
| 1136 | operator>(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1137 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1138 | _GLIBCXX_NOEXCEPTnoexcept |
| 1139 | { return __lhs.base() > __rhs.base(); } |
| 1140 | |
| 1141 | template<typename _Iterator, typename _Container> |
| 1142 | _GLIBCXX20_CONSTEXPR |
| 1143 | inline bool |
| 1144 | operator>(const __normal_iterator<_Iterator, _Container>& __lhs, |
| 1145 | const __normal_iterator<_Iterator, _Container>& __rhs) |
| 1146 | _GLIBCXX_NOEXCEPTnoexcept |
| 1147 | { return __lhs.base() > __rhs.base(); } |
| 1148 | |
| 1149 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1150 | inline bool |
| 1151 | operator<=(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1152 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1153 | _GLIBCXX_NOEXCEPTnoexcept |
| 1154 | { return __lhs.base() <= __rhs.base(); } |
| 1155 | |
| 1156 | template<typename _Iterator, typename _Container> |
| 1157 | _GLIBCXX20_CONSTEXPR |
| 1158 | inline bool |
| 1159 | operator<=(const __normal_iterator<_Iterator, _Container>& __lhs, |
| 1160 | const __normal_iterator<_Iterator, _Container>& __rhs) |
| 1161 | _GLIBCXX_NOEXCEPTnoexcept |
| 1162 | { return __lhs.base() <= __rhs.base(); } |
| 1163 | |
| 1164 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1165 | inline bool |
| 1166 | operator>=(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1167 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1168 | _GLIBCXX_NOEXCEPTnoexcept |
| 1169 | { return __lhs.base() >= __rhs.base(); } |
| 1170 | |
| 1171 | template<typename _Iterator, typename _Container> |
| 1172 | _GLIBCXX20_CONSTEXPR |
| 1173 | inline bool |
| 1174 | operator>=(const __normal_iterator<_Iterator, _Container>& __lhs, |
| 1175 | const __normal_iterator<_Iterator, _Container>& __rhs) |
| 1176 | _GLIBCXX_NOEXCEPTnoexcept |
| 1177 | { return __lhs.base() >= __rhs.base(); } |
| 1178 | #endif // three-way comparison |
| 1179 | |
| 1180 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 1181 | // According to the resolution of DR179 not only the various comparison |
| 1182 | // operators but also operator- must accept mixed iterator/const_iterator |
| 1183 | // parameters. |
| 1184 | template<typename _IteratorL, typename _IteratorR, typename _Container> |
| 1185 | #if __cplusplus201402L >= 201103L |
| 1186 | // DR 685. |
| 1187 | _GLIBCXX20_CONSTEXPR |
| 1188 | inline auto |
| 1189 | operator-(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1190 | const __normal_iterator<_IteratorR, _Container>& __rhs) noexcept |
| 1191 | -> decltype(__lhs.base() - __rhs.base()) |
| 1192 | #else |
| 1193 | inline typename __normal_iterator<_IteratorL, _Container>::difference_type |
| 1194 | operator-(const __normal_iterator<_IteratorL, _Container>& __lhs, |
| 1195 | const __normal_iterator<_IteratorR, _Container>& __rhs) |
| 1196 | #endif |
| 1197 | { return __lhs.base() - __rhs.base(); } |
| 1198 | |
| 1199 | template<typename _Iterator, typename _Container> |
| 1200 | _GLIBCXX20_CONSTEXPR |
| 1201 | inline typename __normal_iterator<_Iterator, _Container>::difference_type |
| 1202 | operator-(const __normal_iterator<_Iterator, _Container>& __lhs, |
| 1203 | const __normal_iterator<_Iterator, _Container>& __rhs) |
| 1204 | _GLIBCXX_NOEXCEPTnoexcept |
| 1205 | { return __lhs.base() - __rhs.base(); } |
| 1206 | |
| 1207 | template<typename _Iterator, typename _Container> |
| 1208 | _GLIBCXX20_CONSTEXPR |
| 1209 | inline __normal_iterator<_Iterator, _Container> |
| 1210 | operator+(typename __normal_iterator<_Iterator, _Container>::difference_type |
| 1211 | __n, const __normal_iterator<_Iterator, _Container>& __i) |
| 1212 | _GLIBCXX_NOEXCEPTnoexcept |
| 1213 | { return __normal_iterator<_Iterator, _Container>(__i.base() + __n); } |
| 1214 | |
| 1215 | _GLIBCXX_END_NAMESPACE_VERSION |
| 1216 | } // namespace |
| 1217 | |
| 1218 | namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default"))) |
| 1219 | { |
| 1220 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
| 1221 | |
| 1222 | template<typename _Iterator, typename _Container> |
| 1223 | _GLIBCXX20_CONSTEXPR |
| 1224 | _Iterator |
| 1225 | __niter_base(__gnu_cxx::__normal_iterator<_Iterator, _Container> __it) |
| 1226 | _GLIBCXX_NOEXCEPT_IF(std::is_nothrow_copy_constructible<_Iterator>::value)noexcept(std::is_nothrow_copy_constructible<_Iterator>:: value) |
| 1227 | { return __it.base(); } |
| 1228 | |
| 1229 | #if __cplusplus201402L >= 201103L |
| 1230 | /** |
| 1231 | * @addtogroup iterators |
| 1232 | * @{ |
| 1233 | */ |
| 1234 | |
| 1235 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1236 | template<semiregular _Sent> |
| 1237 | class move_sentinel |
| 1238 | { |
| 1239 | public: |
| 1240 | constexpr |
| 1241 | move_sentinel() |
| 1242 | noexcept(is_nothrow_default_constructible_v<_Sent>) |
| 1243 | : _M_last() { } |
| 1244 | |
| 1245 | constexpr explicit |
| 1246 | move_sentinel(_Sent __s) |
| 1247 | noexcept(is_nothrow_move_constructible_v<_Sent>) |
| 1248 | : _M_last(std::move(__s)) { } |
| 1249 | |
| 1250 | template<typename _S2> requires convertible_to<const _S2&, _Sent> |
| 1251 | constexpr |
| 1252 | move_sentinel(const move_sentinel<_S2>& __s) |
| 1253 | noexcept(is_nothrow_constructible_v<_Sent, const _S2&>) |
| 1254 | : _M_last(__s.base()) |
| 1255 | { } |
| 1256 | |
| 1257 | template<typename _S2> requires assignable_from<_Sent&, const _S2&> |
| 1258 | constexpr move_sentinel& |
| 1259 | operator=(const move_sentinel<_S2>& __s) |
| 1260 | noexcept(is_nothrow_assignable_v<_Sent, const _S2&>) |
| 1261 | { |
| 1262 | _M_last = __s.base(); |
| 1263 | return *this; |
| 1264 | } |
| 1265 | |
| 1266 | constexpr _Sent |
| 1267 | base() const |
| 1268 | noexcept(is_nothrow_copy_constructible_v<_Sent>) |
| 1269 | { return _M_last; } |
| 1270 | |
| 1271 | private: |
| 1272 | _Sent _M_last; |
| 1273 | }; |
| 1274 | #endif // C++20 |
| 1275 | |
| 1276 | // 24.4.3 Move iterators |
| 1277 | /** |
| 1278 | * Class template move_iterator is an iterator adapter with the same |
| 1279 | * behavior as the underlying iterator except that its dereference |
| 1280 | * operator implicitly converts the value returned by the underlying |
| 1281 | * iterator's dereference operator to an rvalue reference. Some |
| 1282 | * generic algorithms can be called with move iterators to replace |
| 1283 | * copying with moving. |
| 1284 | */ |
| 1285 | template<typename _Iterator> |
| 1286 | class move_iterator |
| 1287 | { |
| 1288 | _Iterator _M_current; |
| 1289 | |
| 1290 | using __traits_type = iterator_traits<_Iterator>; |
| 1291 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1292 | using __base_cat = typename __traits_type::iterator_category; |
| 1293 | #else |
| 1294 | using __base_ref = typename __traits_type::reference; |
| 1295 | #endif |
| 1296 | |
| 1297 | public: |
| 1298 | using iterator_type = _Iterator; |
| 1299 | |
| 1300 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1301 | using iterator_concept = input_iterator_tag; |
| 1302 | using iterator_category |
| 1303 | = __detail::__clamp_iter_cat<__base_cat, random_access_iterator_tag>; |
| 1304 | using value_type = iter_value_t<_Iterator>; |
| 1305 | using difference_type = iter_difference_t<_Iterator>; |
| 1306 | using pointer = _Iterator; |
| 1307 | using reference = iter_rvalue_reference_t<_Iterator>; |
| 1308 | #else |
| 1309 | typedef typename __traits_type::iterator_category iterator_category; |
| 1310 | typedef typename __traits_type::value_type value_type; |
| 1311 | typedef typename __traits_type::difference_type difference_type; |
| 1312 | // NB: DR 680. |
| 1313 | typedef _Iterator pointer; |
| 1314 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 1315 | // 2106. move_iterator wrapping iterators returning prvalues |
| 1316 | typedef typename conditional<is_reference<__base_ref>::value, |
| 1317 | typename remove_reference<__base_ref>::type&&, |
| 1318 | __base_ref>::type reference; |
| 1319 | #endif |
| 1320 | |
| 1321 | _GLIBCXX17_CONSTEXPR |
| 1322 | move_iterator() |
| 1323 | : _M_current() { } |
| 1324 | |
| 1325 | explicit _GLIBCXX17_CONSTEXPR |
| 1326 | move_iterator(iterator_type __i) |
| 1327 | : _M_current(std::move(__i)) { } |
| 1328 | |
| 1329 | template<typename _Iter> |
| 1330 | _GLIBCXX17_CONSTEXPR |
| 1331 | move_iterator(const move_iterator<_Iter>& __i) |
| 1332 | : _M_current(__i.base()) { } |
| 1333 | |
| 1334 | #if __cplusplus201402L <= 201703L |
| 1335 | _GLIBCXX17_CONSTEXPR iterator_type |
| 1336 | base() const |
| 1337 | { return _M_current; } |
| 1338 | #else |
| 1339 | constexpr iterator_type |
| 1340 | base() const & |
| 1341 | #if __cpp_lib_concepts |
| 1342 | requires copy_constructible<iterator_type> |
| 1343 | #endif |
| 1344 | { return _M_current; } |
| 1345 | |
| 1346 | constexpr iterator_type |
| 1347 | base() && |
| 1348 | { return std::move(_M_current); } |
| 1349 | #endif |
| 1350 | |
| 1351 | _GLIBCXX17_CONSTEXPR reference |
| 1352 | operator*() const |
| 1353 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1354 | { return ranges::iter_move(_M_current); } |
| 1355 | #else |
| 1356 | { return static_cast<reference>(*_M_current); } |
| 1357 | #endif |
| 1358 | |
| 1359 | _GLIBCXX17_CONSTEXPR pointer |
| 1360 | operator->() const |
| 1361 | { return _M_current; } |
| 1362 | |
| 1363 | _GLIBCXX17_CONSTEXPR move_iterator& |
| 1364 | operator++() |
| 1365 | { |
| 1366 | ++_M_current; |
| 1367 | return *this; |
| 1368 | } |
| 1369 | |
| 1370 | _GLIBCXX17_CONSTEXPR move_iterator |
| 1371 | operator++(int) |
| 1372 | { |
| 1373 | move_iterator __tmp = *this; |
| 1374 | ++_M_current; |
| 1375 | return __tmp; |
| 1376 | } |
| 1377 | |
| 1378 | #if __cpp_lib_concepts |
| 1379 | constexpr void |
| 1380 | operator++(int) requires (!forward_iterator<_Iterator>) |
| 1381 | { ++_M_current; } |
| 1382 | #endif |
| 1383 | |
| 1384 | _GLIBCXX17_CONSTEXPR move_iterator& |
| 1385 | operator--() |
| 1386 | { |
| 1387 | --_M_current; |
| 1388 | return *this; |
| 1389 | } |
| 1390 | |
| 1391 | _GLIBCXX17_CONSTEXPR move_iterator |
| 1392 | operator--(int) |
| 1393 | { |
| 1394 | move_iterator __tmp = *this; |
| 1395 | --_M_current; |
| 1396 | return __tmp; |
| 1397 | } |
| 1398 | |
| 1399 | _GLIBCXX17_CONSTEXPR move_iterator |
| 1400 | operator+(difference_type __n) const |
| 1401 | { return move_iterator(_M_current + __n); } |
| 1402 | |
| 1403 | _GLIBCXX17_CONSTEXPR move_iterator& |
| 1404 | operator+=(difference_type __n) |
| 1405 | { |
| 1406 | _M_current += __n; |
| 1407 | return *this; |
| 1408 | } |
| 1409 | |
| 1410 | _GLIBCXX17_CONSTEXPR move_iterator |
| 1411 | operator-(difference_type __n) const |
| 1412 | { return move_iterator(_M_current - __n); } |
| 1413 | |
| 1414 | _GLIBCXX17_CONSTEXPR move_iterator& |
| 1415 | operator-=(difference_type __n) |
| 1416 | { |
| 1417 | _M_current -= __n; |
| 1418 | return *this; |
| 1419 | } |
| 1420 | |
| 1421 | _GLIBCXX17_CONSTEXPR reference |
| 1422 | operator[](difference_type __n) const |
| 1423 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1424 | { return ranges::iter_move(_M_current + __n); } |
| 1425 | #else |
| 1426 | { return std::move(_M_current[__n]); } |
| 1427 | #endif |
| 1428 | |
| 1429 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1430 | template<sentinel_for<_Iterator> _Sent> |
| 1431 | friend constexpr bool |
| 1432 | operator==(const move_iterator& __x, const move_sentinel<_Sent>& __y) |
| 1433 | { return __x.base() == __y.base(); } |
| 1434 | |
| 1435 | template<sized_sentinel_for<_Iterator> _Sent> |
| 1436 | friend constexpr iter_difference_t<_Iterator> |
| 1437 | operator-(const move_sentinel<_Sent>& __x, const move_iterator& __y) |
| 1438 | { return __x.base() - __y.base(); } |
| 1439 | |
| 1440 | template<sized_sentinel_for<_Iterator> _Sent> |
| 1441 | friend constexpr iter_difference_t<_Iterator> |
| 1442 | operator-(const move_iterator& __x, const move_sentinel<_Sent>& __y) |
| 1443 | { return __x.base() - __y.base(); } |
| 1444 | |
| 1445 | friend constexpr iter_rvalue_reference_t<_Iterator> |
| 1446 | iter_move(const move_iterator& __i) |
| 1447 | noexcept(noexcept(ranges::iter_move(__i._M_current))) |
| 1448 | { return ranges::iter_move(__i._M_current); } |
| 1449 | |
| 1450 | template<indirectly_swappable<_Iterator> _Iter2> |
| 1451 | friend constexpr void |
| 1452 | iter_swap(const move_iterator& __x, const move_iterator<_Iter2>& __y) |
| 1453 | noexcept(noexcept(ranges::iter_swap(__x._M_current, __y._M_current))) |
| 1454 | { return ranges::iter_swap(__x._M_current, __y._M_current); } |
| 1455 | #endif // C++20 |
| 1456 | }; |
| 1457 | |
| 1458 | template<typename _IteratorL, typename _IteratorR> |
| 1459 | inline _GLIBCXX17_CONSTEXPR bool |
| 1460 | operator==(const move_iterator<_IteratorL>& __x, |
| 1461 | const move_iterator<_IteratorR>& __y) |
| 1462 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1463 | requires requires { { __x.base() == __y.base() } -> convertible_to<bool>; } |
| 1464 | #endif |
| 1465 | { return __x.base() == __y.base(); } |
| 1466 | |
| 1467 | #if __cpp_lib_three_way_comparison |
| 1468 | template<typename _IteratorL, |
| 1469 | three_way_comparable_with<_IteratorL> _IteratorR> |
| 1470 | constexpr compare_three_way_result_t<_IteratorL, _IteratorR> |
| 1471 | operator<=>(const move_iterator<_IteratorL>& __x, |
| 1472 | const move_iterator<_IteratorR>& __y) |
| 1473 | { return __x.base() <=> __y.base(); } |
| 1474 | #else |
| 1475 | template<typename _IteratorL, typename _IteratorR> |
| 1476 | inline _GLIBCXX17_CONSTEXPR bool |
| 1477 | operator!=(const move_iterator<_IteratorL>& __x, |
| 1478 | const move_iterator<_IteratorR>& __y) |
| 1479 | { return !(__x == __y); } |
| 1480 | #endif |
| 1481 | |
| 1482 | template<typename _IteratorL, typename _IteratorR> |
| 1483 | inline _GLIBCXX17_CONSTEXPR bool |
| 1484 | operator<(const move_iterator<_IteratorL>& __x, |
| 1485 | const move_iterator<_IteratorR>& __y) |
| 1486 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1487 | requires requires { { __x.base() < __y.base() } -> convertible_to<bool>; } |
| 1488 | #endif |
| 1489 | { return __x.base() < __y.base(); } |
| 1490 | |
| 1491 | template<typename _IteratorL, typename _IteratorR> |
| 1492 | inline _GLIBCXX17_CONSTEXPR bool |
| 1493 | operator<=(const move_iterator<_IteratorL>& __x, |
| 1494 | const move_iterator<_IteratorR>& __y) |
| 1495 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1496 | requires requires { { __y.base() < __x.base() } -> convertible_to<bool>; } |
| 1497 | #endif |
| 1498 | { return !(__y < __x); } |
| 1499 | |
| 1500 | template<typename _IteratorL, typename _IteratorR> |
| 1501 | inline _GLIBCXX17_CONSTEXPR bool |
| 1502 | operator>(const move_iterator<_IteratorL>& __x, |
| 1503 | const move_iterator<_IteratorR>& __y) |
| 1504 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1505 | requires requires { { __y.base() < __x.base() } -> convertible_to<bool>; } |
| 1506 | #endif |
| 1507 | { return __y < __x; } |
| 1508 | |
| 1509 | template<typename _IteratorL, typename _IteratorR> |
| 1510 | inline _GLIBCXX17_CONSTEXPR bool |
| 1511 | operator>=(const move_iterator<_IteratorL>& __x, |
| 1512 | const move_iterator<_IteratorR>& __y) |
| 1513 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1514 | requires requires { { __x.base() < __y.base() } -> convertible_to<bool>; } |
| 1515 | #endif |
| 1516 | { return !(__x < __y); } |
| 1517 | |
| 1518 | #if ! (__cplusplus201402L > 201703L && __cpp_lib_concepts) |
| 1519 | // Note: See __normal_iterator operators note from Gaby to understand |
| 1520 | // why we have these extra overloads for some move_iterator operators. |
| 1521 | |
| 1522 | // These extra overloads are not needed in C++20, because the ones above |
| 1523 | // are constrained with a requires-clause and so overload resolution will |
| 1524 | // prefer them to greedy unconstrained function templates. |
| 1525 | |
| 1526 | template<typename _Iterator> |
| 1527 | inline _GLIBCXX17_CONSTEXPR bool |
| 1528 | operator==(const move_iterator<_Iterator>& __x, |
| 1529 | const move_iterator<_Iterator>& __y) |
| 1530 | { return __x.base() == __y.base(); } |
| 1531 | |
| 1532 | template<typename _Iterator> |
| 1533 | inline _GLIBCXX17_CONSTEXPR bool |
| 1534 | operator!=(const move_iterator<_Iterator>& __x, |
| 1535 | const move_iterator<_Iterator>& __y) |
| 1536 | { return !(__x == __y); } |
| 1537 | |
| 1538 | template<typename _Iterator> |
| 1539 | inline _GLIBCXX17_CONSTEXPR bool |
| 1540 | operator<(const move_iterator<_Iterator>& __x, |
| 1541 | const move_iterator<_Iterator>& __y) |
| 1542 | { return __x.base() < __y.base(); } |
| 1543 | |
| 1544 | template<typename _Iterator> |
| 1545 | inline _GLIBCXX17_CONSTEXPR bool |
| 1546 | operator<=(const move_iterator<_Iterator>& __x, |
| 1547 | const move_iterator<_Iterator>& __y) |
| 1548 | { return !(__y < __x); } |
| 1549 | |
| 1550 | template<typename _Iterator> |
| 1551 | inline _GLIBCXX17_CONSTEXPR bool |
| 1552 | operator>(const move_iterator<_Iterator>& __x, |
| 1553 | const move_iterator<_Iterator>& __y) |
| 1554 | { return __y < __x; } |
| 1555 | |
| 1556 | template<typename _Iterator> |
| 1557 | inline _GLIBCXX17_CONSTEXPR bool |
| 1558 | operator>=(const move_iterator<_Iterator>& __x, |
| 1559 | const move_iterator<_Iterator>& __y) |
| 1560 | { return !(__x < __y); } |
| 1561 | #endif // ! C++20 |
| 1562 | |
| 1563 | // DR 685. |
| 1564 | template<typename _IteratorL, typename _IteratorR> |
| 1565 | inline _GLIBCXX17_CONSTEXPR auto |
| 1566 | operator-(const move_iterator<_IteratorL>& __x, |
| 1567 | const move_iterator<_IteratorR>& __y) |
| 1568 | -> decltype(__x.base() - __y.base()) |
| 1569 | { return __x.base() - __y.base(); } |
| 1570 | |
| 1571 | template<typename _Iterator> |
| 1572 | inline _GLIBCXX17_CONSTEXPR move_iterator<_Iterator> |
| 1573 | operator+(typename move_iterator<_Iterator>::difference_type __n, |
| 1574 | const move_iterator<_Iterator>& __x) |
| 1575 | { return __x + __n; } |
| 1576 | |
| 1577 | template<typename _Iterator> |
| 1578 | inline _GLIBCXX17_CONSTEXPR move_iterator<_Iterator> |
| 1579 | make_move_iterator(_Iterator __i) |
| 1580 | { return move_iterator<_Iterator>(std::move(__i)); } |
| 1581 | |
| 1582 | template<typename _Iterator, typename _ReturnType |
| 1583 | = typename conditional<__move_if_noexcept_cond |
| 1584 | <typename iterator_traits<_Iterator>::value_type>::value, |
| 1585 | _Iterator, move_iterator<_Iterator>>::type> |
| 1586 | inline _GLIBCXX17_CONSTEXPR _ReturnType |
| 1587 | __make_move_if_noexcept_iterator(_Iterator __i) |
| 1588 | { return _ReturnType(__i); } |
| 1589 | |
| 1590 | // Overload for pointers that matches std::move_if_noexcept more closely, |
| 1591 | // returning a constant iterator when we don't want to move. |
| 1592 | template<typename _Tp, typename _ReturnType |
| 1593 | = typename conditional<__move_if_noexcept_cond<_Tp>::value, |
| 1594 | const _Tp*, move_iterator<_Tp*>>::type> |
| 1595 | inline _GLIBCXX17_CONSTEXPR _ReturnType |
| 1596 | __make_move_if_noexcept_iterator(_Tp* __i) |
| 1597 | { return _ReturnType(__i); } |
| 1598 | |
| 1599 | #if __cplusplus201402L > 201703L && __cpp_lib_concepts |
| 1600 | // [iterators.common] Common iterators |
| 1601 | |
| 1602 | namespace __detail |
| 1603 | { |
| 1604 | template<typename _It> |
| 1605 | concept __common_iter_has_arrow = indirectly_readable<const _It> |
| 1606 | && (requires(const _It& __it) { __it.operator->(); } |
| 1607 | || is_reference_v<iter_reference_t<_It>> |
| 1608 | || constructible_from<iter_value_t<_It>, iter_reference_t<_It>>); |
| 1609 | |
| 1610 | } // namespace __detail |
| 1611 | |
| 1612 | /// An iterator/sentinel adaptor for representing a non-common range. |
| 1613 | template<input_or_output_iterator _It, sentinel_for<_It> _Sent> |
| 1614 | requires (!same_as<_It, _Sent>) && copyable<_It> |
| 1615 | class common_iterator |
| 1616 | { |
| 1617 | template<typename _Tp, typename _Up> |
| 1618 | static constexpr bool |
| 1619 | _S_noexcept1() |
| 1620 | { |
| 1621 | if constexpr (is_trivially_default_constructible_v<_Tp>) |
| 1622 | return is_nothrow_assignable_v<_Tp, _Up>; |
| 1623 | else |
| 1624 | return is_nothrow_constructible_v<_Tp, _Up>; |
| 1625 | } |
| 1626 | |
| 1627 | template<typename _It2, typename _Sent2> |
| 1628 | static constexpr bool |
| 1629 | _S_noexcept() |
| 1630 | { return _S_noexcept1<_It, _It2>() && _S_noexcept1<_Sent, _Sent2>(); } |
| 1631 | |
| 1632 | class _Proxy |
| 1633 | { |
| 1634 | iter_value_t<_It> _M_keep; |
| 1635 | |
| 1636 | _Proxy(iter_reference_t<_It>&& __x) |
| 1637 | : _M_keep(std::move(__x)) { } |
| 1638 | |
| 1639 | friend class common_iterator; |
| 1640 | |
| 1641 | public: |
| 1642 | const iter_value_t<_It>* |
| 1643 | operator->() const |
| 1644 | { return std::__addressof(_M_keep); } |
| 1645 | }; |
| 1646 | |
| 1647 | public: |
| 1648 | constexpr |
| 1649 | common_iterator() |
| 1650 | noexcept(is_nothrow_default_constructible_v<_It>) |
| 1651 | : _M_it(), _M_index(0) |
| 1652 | { } |
| 1653 | |
| 1654 | constexpr |
| 1655 | common_iterator(_It __i) |
| 1656 | noexcept(is_nothrow_move_constructible_v<_It>) |
| 1657 | : _M_it(std::move(__i)), _M_index(0) |
| 1658 | { } |
| 1659 | |
| 1660 | constexpr |
| 1661 | common_iterator(_Sent __s) |
| 1662 | noexcept(is_nothrow_move_constructible_v<_Sent>) |
| 1663 | : _M_sent(std::move(__s)), _M_index(1) |
| 1664 | { } |
| 1665 | |
| 1666 | template<typename _It2, typename _Sent2> |
| 1667 | requires convertible_to<const _It2&, _It> |
| 1668 | && convertible_to<const _Sent2&, _Sent> |
| 1669 | constexpr |
| 1670 | common_iterator(const common_iterator<_It2, _Sent2>& __x) |
| 1671 | noexcept(_S_noexcept<const _It2&, const _Sent2&>()) |
| 1672 | : _M_valueless(), _M_index(__x._M_index) |
| 1673 | { |
| 1674 | if (_M_index == 0) |
| 1675 | { |
| 1676 | if constexpr (is_trivially_default_constructible_v<_It>) |
| 1677 | _M_it = std::move(__x._M_it); |
| 1678 | else |
| 1679 | ::new((void*)std::__addressof(_M_it)) _It(__x._M_it); |
| 1680 | } |
| 1681 | else if (_M_index == 1) |
| 1682 | { |
| 1683 | if constexpr (is_trivially_default_constructible_v<_Sent>) |
| 1684 | _M_sent = std::move(__x._M_sent); |
| 1685 | else |
| 1686 | ::new((void*)std::__addressof(_M_sent)) _Sent(__x._M_sent); |
| 1687 | } |
| 1688 | } |
| 1689 | |
| 1690 | constexpr |
| 1691 | common_iterator(const common_iterator& __x) |
| 1692 | noexcept(_S_noexcept<const _It&, const _Sent&>()) |
| 1693 | : _M_valueless(), _M_index(__x._M_index) |
| 1694 | { |
| 1695 | if (_M_index == 0) |
| 1696 | { |
| 1697 | if constexpr (is_trivially_default_constructible_v<_It>) |
| 1698 | _M_it = std::move(__x._M_it); |
| 1699 | else |
| 1700 | ::new((void*)std::__addressof(_M_it)) _It(__x._M_it); |
| 1701 | } |
| 1702 | else if (_M_index == 1) |
| 1703 | { |
| 1704 | if constexpr (is_trivially_default_constructible_v<_Sent>) |
| 1705 | _M_sent = std::move(__x._M_sent); |
| 1706 | else |
| 1707 | ::new((void*)std::__addressof(_M_sent)) _Sent(__x._M_sent); |
| 1708 | } |
| 1709 | } |
| 1710 | |
| 1711 | common_iterator& |
| 1712 | operator=(const common_iterator& __x) |
| 1713 | noexcept(is_nothrow_copy_assignable_v<_It> |
| 1714 | && is_nothrow_copy_assignable_v<_Sent> |
| 1715 | && is_nothrow_copy_constructible_v<_It> |
| 1716 | && is_nothrow_copy_constructible_v<_Sent>) |
| 1717 | { |
| 1718 | return this->operator=<_It, _Sent>(__x); |
| 1719 | } |
| 1720 | |
| 1721 | template<typename _It2, typename _Sent2> |
| 1722 | requires convertible_to<const _It2&, _It> |
| 1723 | && convertible_to<const _Sent2&, _Sent> |
| 1724 | && assignable_from<_It&, const _It2&> |
| 1725 | && assignable_from<_Sent&, const _Sent2&> |
| 1726 | common_iterator& |
| 1727 | operator=(const common_iterator<_It2, _Sent2>& __x) |
| 1728 | noexcept(is_nothrow_constructible_v<_It, const _It2&> |
| 1729 | && is_nothrow_constructible_v<_Sent, const _Sent2&> |
| 1730 | && is_nothrow_assignable_v<_It, const _It2&> |
| 1731 | && is_nothrow_assignable_v<_Sent, const _Sent2&>) |
| 1732 | { |
| 1733 | switch(_M_index << 2 | __x._M_index) |
| 1734 | { |
| 1735 | case 0b0000: |
| 1736 | _M_it = __x._M_it; |
| 1737 | break; |
| 1738 | case 0b0101: |
| 1739 | _M_sent = __x._M_sent; |
| 1740 | break; |
| 1741 | case 0b0001: |
| 1742 | _M_it.~_It(); |
| 1743 | _M_index = -1; |
| 1744 | [[fallthrough]]; |
| 1745 | case 0b1001: |
| 1746 | ::new((void*)std::__addressof(_M_sent)) _Sent(__x._M_sent); |
| 1747 | _M_index = 1; |
| 1748 | break; |
| 1749 | case 0b0100: |
| 1750 | _M_sent.~_Sent(); |
| 1751 | _M_index = -1; |
| 1752 | [[fallthrough]]; |
| 1753 | case 0b1000: |
| 1754 | ::new((void*)std::__addressof(_M_it)) _It(__x._M_it); |
| 1755 | _M_index = 0; |
| 1756 | break; |
| 1757 | default: |
| 1758 | __glibcxx_assert(__x._M_has_value()); |
| 1759 | __builtin_unreachable(); |
| 1760 | } |
| 1761 | return *this; |
| 1762 | } |
| 1763 | |
| 1764 | ~common_iterator() |
| 1765 | { |
| 1766 | switch (_M_index) |
| 1767 | { |
| 1768 | case 0: |
| 1769 | _M_it.~_It(); |
| 1770 | break; |
| 1771 | case 1: |
| 1772 | _M_sent.~_Sent(); |
| 1773 | break; |
| 1774 | } |
| 1775 | } |
| 1776 | |
| 1777 | decltype(auto) |
| 1778 | operator*() |
| 1779 | { |
| 1780 | __glibcxx_assert(_M_index == 0); |
| 1781 | return *_M_it; |
| 1782 | } |
| 1783 | |
| 1784 | decltype(auto) |
| 1785 | operator*() const requires __detail::__dereferenceable<const _It> |
| 1786 | { |
| 1787 | __glibcxx_assert(_M_index == 0); |
| 1788 | return *_M_it; |
| 1789 | } |
| 1790 | |
| 1791 | decltype(auto) |
| 1792 | operator->() const requires __detail::__common_iter_has_arrow<_It> |
| 1793 | { |
| 1794 | __glibcxx_assert(_M_index == 0); |
| 1795 | if constexpr (is_pointer_v<_It> || requires { _M_it.operator->(); }) |
| 1796 | return _M_it; |
| 1797 | else if constexpr (is_reference_v<iter_reference_t<_It>>) |
| 1798 | { |
| 1799 | auto&& __tmp = *_M_it; |
| 1800 | return std::__addressof(__tmp); |
| 1801 | } |
| 1802 | else |
| 1803 | return _Proxy{*_M_it}; |
| 1804 | } |
| 1805 | |
| 1806 | common_iterator& |
| 1807 | operator++() |
| 1808 | { |
| 1809 | __glibcxx_assert(_M_index == 0); |
| 1810 | ++_M_it; |
| 1811 | return *this; |
| 1812 | } |
| 1813 | |
| 1814 | decltype(auto) |
| 1815 | operator++(int) |
| 1816 | { |
| 1817 | __glibcxx_assert(_M_index == 0); |
| 1818 | if constexpr (forward_iterator<_It>) |
| 1819 | { |
| 1820 | common_iterator __tmp = *this; |
| 1821 | ++*this; |
| 1822 | return __tmp; |
| 1823 | } |
| 1824 | else |
| 1825 | return _M_it++; |
| 1826 | } |
| 1827 | |
| 1828 | template<typename _It2, sentinel_for<_It> _Sent2> |
| 1829 | requires sentinel_for<_Sent, _It2> |
| 1830 | friend bool |
| 1831 | operator==(const common_iterator& __x, |
| 1832 | const common_iterator<_It2, _Sent2>& __y) |
| 1833 | { |
| 1834 | switch(__x._M_index << 2 | __y._M_index) |
| 1835 | { |
| 1836 | case 0b0000: |
| 1837 | case 0b0101: |
| 1838 | return true; |
| 1839 | case 0b0001: |
| 1840 | return __x._M_it == __y._M_sent; |
| 1841 | case 0b0100: |
| 1842 | return __x._M_sent == __y._M_it; |
| 1843 | default: |
| 1844 | __glibcxx_assert(__x._M_has_value()); |
| 1845 | __glibcxx_assert(__y._M_has_value()); |
| 1846 | __builtin_unreachable(); |
| 1847 | } |
| 1848 | } |
| 1849 | |
| 1850 | template<typename _It2, sentinel_for<_It> _Sent2> |
| 1851 | requires sentinel_for<_Sent, _It2> && equality_comparable_with<_It, _It2> |
| 1852 | friend bool |
| 1853 | operator==(const common_iterator& __x, |
| 1854 | const common_iterator<_It2, _Sent2>& __y) |
| 1855 | { |
| 1856 | switch(__x._M_index << 2 | __y._M_index) |
| 1857 | { |
| 1858 | case 0b0101: |
| 1859 | return true; |
| 1860 | case 0b0000: |
| 1861 | return __x._M_it == __y._M_it; |
| 1862 | case 0b0001: |
| 1863 | return __x._M_it == __y._M_sent; |
| 1864 | case 0b0100: |
| 1865 | return __x._M_sent == __y._M_it; |
| 1866 | default: |
| 1867 | __glibcxx_assert(__x._M_has_value()); |
| 1868 | __glibcxx_assert(__y._M_has_value()); |
| 1869 | __builtin_unreachable(); |
| 1870 | } |
| 1871 | } |
| 1872 | |
| 1873 | template<sized_sentinel_for<_It> _It2, sized_sentinel_for<_It> _Sent2> |
| 1874 | requires sized_sentinel_for<_Sent, _It2> |
| 1875 | friend iter_difference_t<_It2> |
| 1876 | operator-(const common_iterator& __x, |
| 1877 | const common_iterator<_It2, _Sent2>& __y) |
| 1878 | { |
| 1879 | switch(__x._M_index << 2 | __y._M_index) |
| 1880 | { |
| 1881 | case 0b0101: |
| 1882 | return 0; |
| 1883 | case 0b0000: |
| 1884 | return __x._M_it - __y._M_it; |
| 1885 | case 0b0001: |
| 1886 | return __x._M_it - __y._M_sent; |
| 1887 | case 0b0100: |
| 1888 | return __x._M_sent - __y._M_it; |
| 1889 | default: |
| 1890 | __glibcxx_assert(__x._M_has_value()); |
| 1891 | __glibcxx_assert(__y._M_has_value()); |
| 1892 | __builtin_unreachable(); |
| 1893 | } |
| 1894 | } |
| 1895 | |
| 1896 | friend iter_rvalue_reference_t<_It> |
| 1897 | iter_move(const common_iterator& __i) |
| 1898 | noexcept(noexcept(ranges::iter_move(std::declval<const _It&>()))) |
| 1899 | requires input_iterator<_It> |
| 1900 | { |
| 1901 | __glibcxx_assert(__i._M_index == 0); |
| 1902 | return ranges::iter_move(__i._M_it); |
| 1903 | } |
| 1904 | |
| 1905 | template<indirectly_swappable<_It> _It2, typename _Sent2> |
| 1906 | friend void |
| 1907 | iter_swap(const common_iterator& __x, |
| 1908 | const common_iterator<_It2, _Sent2>& __y) |
| 1909 | noexcept(noexcept(ranges::iter_swap(std::declval<const _It&>(), |
| 1910 | std::declval<const _It2&>()))) |
| 1911 | { |
| 1912 | __glibcxx_assert(__x._M_index == 0); |
| 1913 | __glibcxx_assert(__y._M_index == 0); |
| 1914 | return ranges::iter_swap(__x._M_it, __y._M_it); |
| 1915 | } |
| 1916 | |
| 1917 | private: |
| 1918 | template<input_or_output_iterator _It2, sentinel_for<_It2> _Sent2> |
| 1919 | friend class common_iterator; |
| 1920 | |
| 1921 | bool _M_has_value() const noexcept { return _M_index < 2; } |
| 1922 | |
| 1923 | union |
| 1924 | { |
| 1925 | _It _M_it; |
| 1926 | _Sent _M_sent; |
| 1927 | unsigned char _M_valueless; |
| 1928 | }; |
| 1929 | unsigned char _M_index; // 0==_M_it, 1==_M_sent, 2==valueless |
| 1930 | }; |
| 1931 | |
| 1932 | template<typename _It, typename _Sent> |
| 1933 | struct incrementable_traits<common_iterator<_It, _Sent>> |
| 1934 | { |
| 1935 | using difference_type = iter_difference_t<_It>; |
| 1936 | }; |
| 1937 | |
| 1938 | template<input_iterator _It, typename _Sent> |
| 1939 | struct iterator_traits<common_iterator<_It, _Sent>> |
| 1940 | { |
| 1941 | private: |
| 1942 | template<typename _Iter> |
| 1943 | struct __ptr |
| 1944 | { |
| 1945 | using type = void; |
| 1946 | }; |
| 1947 | |
| 1948 | template<typename _Iter> |
| 1949 | requires __detail::__common_iter_has_arrow<_Iter> |
| 1950 | struct __ptr<_Iter> |
| 1951 | { |
| 1952 | using _CIter = common_iterator<_Iter, _Sent>; |
| 1953 | using type = decltype(std::declval<const _CIter&>().operator->()); |
| 1954 | }; |
| 1955 | |
| 1956 | public: |
| 1957 | using iterator_concept = conditional_t<forward_iterator<_It>, |
| 1958 | forward_iterator_tag, input_iterator_tag>; |
| 1959 | using iterator_category = __detail::__clamp_iter_cat< |
| 1960 | typename iterator_traits<_It>::iterator_category, |
| 1961 | forward_iterator_tag, input_iterator_tag>; |
| 1962 | using value_type = iter_value_t<_It>; |
| 1963 | using difference_type = iter_difference_t<_It>; |
| 1964 | using pointer = typename __ptr<_It>::type; |
| 1965 | using reference = iter_reference_t<_It>; |
| 1966 | }; |
| 1967 | |
| 1968 | // [iterators.counted] Counted iterators |
| 1969 | |
| 1970 | /// An iterator adaptor that keeps track of the distance to the end. |
| 1971 | template<input_or_output_iterator _It> |
| 1972 | class counted_iterator |
| 1973 | { |
| 1974 | public: |
| 1975 | using iterator_type = _It; |
| 1976 | |
| 1977 | constexpr counted_iterator() = default; |
| 1978 | |
| 1979 | constexpr |
| 1980 | counted_iterator(_It __i, iter_difference_t<_It> __n) |
| 1981 | : _M_current(std::move(__i)), _M_length(__n) |
| 1982 | { __glibcxx_assert(__n >= 0); } |
| 1983 | |
| 1984 | template<typename _It2> |
| 1985 | requires convertible_to<const _It2&, _It> |
| 1986 | constexpr |
| 1987 | counted_iterator(const counted_iterator<_It2>& __x) |
| 1988 | : _M_current(__x._M_current), _M_length(__x._M_length) |
| 1989 | { } |
| 1990 | |
| 1991 | template<typename _It2> |
| 1992 | requires assignable_from<_It&, const _It2&> |
| 1993 | constexpr counted_iterator& |
| 1994 | operator=(const counted_iterator<_It2>& __x) |
| 1995 | { |
| 1996 | _M_current = __x._M_current; |
| 1997 | _M_length = __x._M_length; |
| 1998 | return *this; |
| 1999 | } |
| 2000 | |
| 2001 | constexpr _It |
| 2002 | base() const & |
| 2003 | noexcept(is_nothrow_copy_constructible_v<_It>) |
| 2004 | requires copy_constructible<_It> |
| 2005 | { return _M_current; } |
| 2006 | |
| 2007 | constexpr _It |
| 2008 | base() && |
| 2009 | noexcept(is_nothrow_move_constructible_v<_It>) |
| 2010 | { return std::move(_M_current); } |
| 2011 | |
| 2012 | constexpr iter_difference_t<_It> |
| 2013 | count() const noexcept { return _M_length; } |
| 2014 | |
| 2015 | constexpr decltype(auto) |
| 2016 | operator*() |
| 2017 | noexcept(noexcept(*_M_current)) |
| 2018 | { return *_M_current; } |
| 2019 | |
| 2020 | constexpr decltype(auto) |
| 2021 | operator*() const |
| 2022 | noexcept(noexcept(*_M_current)) |
| 2023 | requires __detail::__dereferenceable<const _It> |
| 2024 | { return *_M_current; } |
| 2025 | |
| 2026 | constexpr counted_iterator& |
| 2027 | operator++() |
| 2028 | { |
| 2029 | __glibcxx_assert(_M_length > 0); |
| 2030 | ++_M_current; |
| 2031 | --_M_length; |
| 2032 | return *this; |
| 2033 | } |
| 2034 | |
| 2035 | decltype(auto) |
| 2036 | operator++(int) |
| 2037 | { |
| 2038 | __glibcxx_assert(_M_length > 0); |
| 2039 | --_M_length; |
| 2040 | __tryif (true) |
| 2041 | { |
| 2042 | return _M_current++; |
| 2043 | } __catch(...)if (false) { |
| 2044 | ++_M_length; |
| 2045 | __throw_exception_again; |
| 2046 | } |
| 2047 | |
| 2048 | } |
| 2049 | |
| 2050 | constexpr counted_iterator |
| 2051 | operator++(int) requires forward_iterator<_It> |
| 2052 | { |
| 2053 | auto __tmp = *this; |
| 2054 | ++*this; |
| 2055 | return __tmp; |
| 2056 | } |
| 2057 | |
| 2058 | constexpr counted_iterator& |
| 2059 | operator--() requires bidirectional_iterator<_It> |
| 2060 | { |
| 2061 | --_M_current; |
| 2062 | ++_M_length; |
| 2063 | return *this; |
| 2064 | } |
| 2065 | |
| 2066 | constexpr counted_iterator |
| 2067 | operator--(int) requires bidirectional_iterator<_It> |
| 2068 | { |
| 2069 | auto __tmp = *this; |
| 2070 | --*this; |
| 2071 | return __tmp; |
| 2072 | } |
| 2073 | |
| 2074 | constexpr counted_iterator |
| 2075 | operator+(iter_difference_t<_It> __n) const |
| 2076 | requires random_access_iterator<_It> |
| 2077 | { return counted_iterator(_M_current + __n, _M_length - __n); } |
| 2078 | |
| 2079 | friend constexpr counted_iterator |
| 2080 | operator+(iter_difference_t<_It> __n, const counted_iterator& __x) |
| 2081 | requires random_access_iterator<_It> |
| 2082 | { return __x + __n; } |
| 2083 | |
| 2084 | constexpr counted_iterator& |
| 2085 | operator+=(iter_difference_t<_It> __n) |
| 2086 | requires random_access_iterator<_It> |
| 2087 | { |
| 2088 | __glibcxx_assert(__n <= _M_length); |
| 2089 | _M_current += __n; |
| 2090 | _M_length -= __n; |
| 2091 | return *this; |
| 2092 | } |
| 2093 | |
| 2094 | constexpr counted_iterator |
| 2095 | operator-(iter_difference_t<_It> __n) const |
| 2096 | requires random_access_iterator<_It> |
| 2097 | { return counted_iterator(_M_current - __n, _M_length + __n); } |
| 2098 | |
| 2099 | template<common_with<_It> _It2> |
| 2100 | friend constexpr iter_difference_t<_It2> |
| 2101 | operator-(const counted_iterator& __x, |
| 2102 | const counted_iterator<_It2>& __y) |
| 2103 | { return __y._M_length - __x._M_length; } |
| 2104 | |
| 2105 | friend constexpr iter_difference_t<_It> |
| 2106 | operator-(const counted_iterator& __x, default_sentinel_t) |
| 2107 | { return -__x._M_length; } |
| 2108 | |
| 2109 | friend constexpr iter_difference_t<_It> |
| 2110 | operator-(default_sentinel_t, const counted_iterator& __y) |
| 2111 | { return __y._M_length; } |
| 2112 | |
| 2113 | constexpr counted_iterator& |
| 2114 | operator-=(iter_difference_t<_It> __n) |
| 2115 | requires random_access_iterator<_It> |
| 2116 | { |
| 2117 | __glibcxx_assert(-__n <= _M_length); |
| 2118 | _M_current -= __n; |
| 2119 | _M_length += __n; |
| 2120 | return *this; |
| 2121 | } |
| 2122 | |
| 2123 | constexpr decltype(auto) |
| 2124 | operator[](iter_difference_t<_It> __n) const |
| 2125 | noexcept(noexcept(_M_current[__n])) |
| 2126 | requires random_access_iterator<_It> |
| 2127 | { |
| 2128 | __glibcxx_assert(__n < _M_length); |
| 2129 | return _M_current[__n]; |
| 2130 | } |
| 2131 | |
| 2132 | template<common_with<_It> _It2> |
| 2133 | friend constexpr bool |
| 2134 | operator==(const counted_iterator& __x, |
| 2135 | const counted_iterator<_It2>& __y) |
| 2136 | { return __x._M_length == __y._M_length; } |
| 2137 | |
| 2138 | friend constexpr bool |
| 2139 | operator==(const counted_iterator& __x, default_sentinel_t) |
| 2140 | { return __x._M_length == 0; } |
| 2141 | |
| 2142 | template<common_with<_It> _It2> |
| 2143 | friend constexpr strong_ordering |
| 2144 | operator<=>(const counted_iterator& __x, |
| 2145 | const counted_iterator<_It2>& __y) |
| 2146 | { return __y._M_length <=> __x._M_length; } |
| 2147 | |
| 2148 | friend constexpr iter_rvalue_reference_t<_It> |
| 2149 | iter_move(const counted_iterator& __i) |
| 2150 | noexcept(noexcept(ranges::iter_move(__i._M_current))) |
| 2151 | requires input_iterator<_It> |
| 2152 | { return ranges::iter_move(__i._M_current); } |
| 2153 | |
| 2154 | template<indirectly_swappable<_It> _It2> |
| 2155 | friend constexpr void |
| 2156 | iter_swap(const counted_iterator& __x, |
| 2157 | const counted_iterator<_It2>& __y) |
| 2158 | noexcept(noexcept(ranges::iter_swap(__x._M_current, __y._M_current))) |
| 2159 | { ranges::iter_swap(__x._M_current, __y._M_current); } |
| 2160 | |
| 2161 | private: |
| 2162 | template<input_or_output_iterator _It2> friend class counted_iterator; |
| 2163 | |
| 2164 | _It _M_current = _It(); |
| 2165 | iter_difference_t<_It> _M_length = 0; |
| 2166 | }; |
| 2167 | |
| 2168 | template<typename _It> |
| 2169 | struct incrementable_traits<counted_iterator<_It>> |
| 2170 | { |
| 2171 | using difference_type = iter_difference_t<_It>; |
| 2172 | }; |
| 2173 | |
| 2174 | template<input_iterator _It> |
| 2175 | struct iterator_traits<counted_iterator<_It>> : iterator_traits<_It> |
| 2176 | { |
| 2177 | using pointer = void; |
| 2178 | }; |
| 2179 | #endif // C++20 |
| 2180 | |
| 2181 | // @} group iterators |
| 2182 | |
| 2183 | template<typename _Iterator> |
| 2184 | auto |
| 2185 | __niter_base(move_iterator<_Iterator> __it) |
| 2186 | -> decltype(make_move_iterator(__niter_base(__it.base()))) |
| 2187 | { return make_move_iterator(__niter_base(__it.base())); } |
| 2188 | |
| 2189 | template<typename _Iterator> |
| 2190 | struct __is_move_iterator<move_iterator<_Iterator> > |
| 2191 | { |
| 2192 | enum { __value = 1 }; |
| 2193 | typedef __true_type __type; |
| 2194 | }; |
| 2195 | |
| 2196 | template<typename _Iterator> |
| 2197 | auto |
| 2198 | __miter_base(move_iterator<_Iterator> __it) |
| 2199 | -> decltype(__miter_base(__it.base())) |
| 2200 | { return __miter_base(__it.base()); } |
| 2201 | |
| 2202 | #define _GLIBCXX_MAKE_MOVE_ITERATOR(_Iter)std::make_move_iterator(_Iter) std::make_move_iterator(_Iter) |
| 2203 | #define _GLIBCXX_MAKE_MOVE_IF_NOEXCEPT_ITERATOR(_Iter)std::__make_move_if_noexcept_iterator(_Iter) \ |
| 2204 | std::__make_move_if_noexcept_iterator(_Iter) |
| 2205 | #else |
| 2206 | #define _GLIBCXX_MAKE_MOVE_ITERATOR(_Iter)std::make_move_iterator(_Iter) (_Iter) |
| 2207 | #define _GLIBCXX_MAKE_MOVE_IF_NOEXCEPT_ITERATOR(_Iter)std::__make_move_if_noexcept_iterator(_Iter) (_Iter) |
| 2208 | #endif // C++11 |
| 2209 | |
| 2210 | #if __cpp_deduction_guides >= 201606 |
| 2211 | // These helper traits are used for deduction guides |
| 2212 | // of associative containers. |
| 2213 | template<typename _InputIterator> |
| 2214 | using __iter_key_t = remove_const_t< |
| 2215 | typename iterator_traits<_InputIterator>::value_type::first_type>; |
| 2216 | |
| 2217 | template<typename _InputIterator> |
| 2218 | using __iter_val_t = |
| 2219 | typename iterator_traits<_InputIterator>::value_type::second_type; |
| 2220 | |
| 2221 | template<typename _T1, typename _T2> |
| 2222 | struct pair; |
| 2223 | |
| 2224 | template<typename _InputIterator> |
| 2225 | using __iter_to_alloc_t = |
| 2226 | pair<add_const_t<__iter_key_t<_InputIterator>>, |
| 2227 | __iter_val_t<_InputIterator>>; |
| 2228 | #endif // __cpp_deduction_guides |
| 2229 | |
| 2230 | _GLIBCXX_END_NAMESPACE_VERSION |
| 2231 | } // namespace |
| 2232 | |
| 2233 | #ifdef _GLIBCXX_DEBUG |
| 2234 | # include <debug/stl_iterator.h> |
| 2235 | #endif |
| 2236 | |
| 2237 | #endif |
| 1 | //===- llvm/Analysis/VectorUtils.h - Vector utilities -----------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines some vectorizer utilities. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef LLVM_ANALYSIS_VECTORUTILS_H |
| 14 | #define LLVM_ANALYSIS_VECTORUTILS_H |
| 15 | |
| 16 | #include "llvm/ADT/MapVector.h" |
| 17 | #include "llvm/ADT/SmallVector.h" |
| 18 | #include "llvm/Analysis/LoopAccessAnalysis.h" |
| 19 | #include "llvm/Support/CheckedArithmetic.h" |
| 20 | |
| 21 | namespace llvm { |
| 22 | class TargetLibraryInfo; |
| 23 | |
| 24 | /// Describes the type of Parameters |
| 25 | enum class VFParamKind { |
| 26 | Vector, // No semantic information. |
| 27 | OMP_Linear, // declare simd linear(i) |
| 28 | OMP_LinearRef, // declare simd linear(ref(i)) |
| 29 | OMP_LinearVal, // declare simd linear(val(i)) |
| 30 | OMP_LinearUVal, // declare simd linear(uval(i)) |
| 31 | OMP_LinearPos, // declare simd linear(i:c) uniform(c) |
| 32 | OMP_LinearValPos, // declare simd linear(val(i:c)) uniform(c) |
| 33 | OMP_LinearRefPos, // declare simd linear(ref(i:c)) uniform(c) |
| 34 | OMP_LinearUValPos, // declare simd linear(uval(i:c)) uniform(c) |
| 35 | OMP_Uniform, // declare simd uniform(i) |
| 36 | GlobalPredicate, // Global logical predicate that acts on all lanes |
| 37 | // of the input and output mask concurrently. For |
| 38 | // example, it is implied by the `M` token in the |
| 39 | // Vector Function ABI mangled name. |
| 40 | Unknown |
| 41 | }; |
| 42 | |
| 43 | /// Describes the type of Instruction Set Architecture |
| 44 | enum class VFISAKind { |
| 45 | AdvancedSIMD, // AArch64 Advanced SIMD (NEON) |
| 46 | SVE, // AArch64 Scalable Vector Extension |
| 47 | SSE, // x86 SSE |
| 48 | AVX, // x86 AVX |
| 49 | AVX2, // x86 AVX2 |
| 50 | AVX512, // x86 AVX512 |
| 51 | LLVM, // LLVM internal ISA for functions that are not |
| 52 | // attached to an existing ABI via name mangling. |
| 53 | Unknown // Unknown ISA |
| 54 | }; |
| 55 | |
| 56 | /// Encapsulates information needed to describe a parameter. |
| 57 | /// |
| 58 | /// The description of the parameter is not linked directly to |
| 59 | /// OpenMP or any other vector function description. This structure |
| 60 | /// is extendible to handle other paradigms that describe vector |
| 61 | /// functions and their parameters. |
| 62 | struct VFParameter { |
| 63 | unsigned ParamPos; // Parameter Position in Scalar Function. |
| 64 | VFParamKind ParamKind; // Kind of Parameter. |
| 65 | int LinearStepOrPos = 0; // Step or Position of the Parameter. |
| 66 | Align Alignment = Align(); // Optional alignment in bytes, defaulted to 1. |
| 67 | |
| 68 | // Comparison operator. |
| 69 | bool operator==(const VFParameter &Other) const { |
| 70 | return std::tie(ParamPos, ParamKind, LinearStepOrPos, Alignment) == |
| 71 | std::tie(Other.ParamPos, Other.ParamKind, Other.LinearStepOrPos, |
| 72 | Other.Alignment); |
| 73 | } |
| 74 | }; |
| 75 | |
| 76 | /// Contains the information about the kind of vectorization |
| 77 | /// available. |
| 78 | /// |
| 79 | /// This object in independent on the paradigm used to |
| 80 | /// represent vector functions. in particular, it is not attached to |
| 81 | /// any target-specific ABI. |
| 82 | struct VFShape { |
| 83 | ElementCount VF; // Vectorization factor. |
| 84 | SmallVector<VFParameter, 8> Parameters; // List of parameter information. |
| 85 | // Comparison operator. |
| 86 | bool operator==(const VFShape &Other) const { |
| 87 | return std::tie(VF, Parameters) == std::tie(Other.VF, Other.Parameters); |
| 88 | } |
| 89 | |
| 90 | /// Update the parameter in position P.ParamPos to P. |
| 91 | void updateParam(VFParameter P) { |
| 92 | assert(P.ParamPos < Parameters.size() && "Invalid parameter position.")(static_cast <bool> (P.ParamPos < Parameters.size() && "Invalid parameter position.") ? void (0) : __assert_fail ("P.ParamPos < Parameters.size() && \"Invalid parameter position.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 92, __extension__ __PRETTY_FUNCTION__)); |
| 93 | Parameters[P.ParamPos] = P; |
| 94 | assert(hasValidParameterList() && "Invalid parameter list")(static_cast <bool> (hasValidParameterList() && "Invalid parameter list") ? void (0) : __assert_fail ("hasValidParameterList() && \"Invalid parameter list\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 94, __extension__ __PRETTY_FUNCTION__)); |
| 95 | } |
| 96 | |
| 97 | // Retrieve the VFShape that can be used to map a (scalar) function to itself, |
| 98 | // with VF = 1. |
| 99 | static VFShape getScalarShape(const CallInst &CI) { |
| 100 | return VFShape::get(CI, ElementCount::getFixed(1), |
| 101 | /*HasGlobalPredicate*/ false); |
| 102 | } |
| 103 | |
| 104 | // Retrieve the basic vectorization shape of the function, where all |
| 105 | // parameters are mapped to VFParamKind::Vector with \p EC |
| 106 | // lanes. Specifies whether the function has a Global Predicate |
| 107 | // argument via \p HasGlobalPred. |
| 108 | static VFShape get(const CallInst &CI, ElementCount EC, bool HasGlobalPred) { |
| 109 | SmallVector<VFParameter, 8> Parameters; |
| 110 | for (unsigned I = 0; I < CI.arg_size(); ++I) |
| 111 | Parameters.push_back(VFParameter({I, VFParamKind::Vector})); |
| 112 | if (HasGlobalPred) |
| 113 | Parameters.push_back( |
| 114 | VFParameter({CI.arg_size(), VFParamKind::GlobalPredicate})); |
| 115 | |
| 116 | return {EC, Parameters}; |
| 117 | } |
| 118 | /// Sanity check on the Parameters in the VFShape. |
| 119 | bool hasValidParameterList() const; |
| 120 | }; |
| 121 | |
| 122 | /// Holds the VFShape for a specific scalar to vector function mapping. |
| 123 | struct VFInfo { |
| 124 | VFShape Shape; /// Classification of the vector function. |
| 125 | std::string ScalarName; /// Scalar Function Name. |
| 126 | std::string VectorName; /// Vector Function Name associated to this VFInfo. |
| 127 | VFISAKind ISA; /// Instruction Set Architecture. |
| 128 | }; |
| 129 | |
| 130 | namespace VFABI { |
| 131 | /// LLVM Internal VFABI ISA token for vector functions. |
| 132 | static constexpr char const *_LLVM_ = "_LLVM_"; |
| 133 | /// Prefix for internal name redirection for vector function that |
| 134 | /// tells the compiler to scalarize the call using the scalar name |
| 135 | /// of the function. For example, a mangled name like |
| 136 | /// `_ZGV_LLVM_N2v_foo(_LLVM_Scalarize_foo)` would tell the |
| 137 | /// vectorizer to vectorize the scalar call `foo`, and to scalarize |
| 138 | /// it once vectorization is done. |
| 139 | static constexpr char const *_LLVM_Scalarize_ = "_LLVM_Scalarize_"; |
| 140 | |
| 141 | /// Function to construct a VFInfo out of a mangled names in the |
| 142 | /// following format: |
| 143 | /// |
| 144 | /// <VFABI_name>{(<redirection>)} |
| 145 | /// |
| 146 | /// where <VFABI_name> is the name of the vector function, mangled according |
| 147 | /// to the rules described in the Vector Function ABI of the target vector |
| 148 | /// extension (or <isa> from now on). The <VFABI_name> is in the following |
| 149 | /// format: |
| 150 | /// |
| 151 | /// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)] |
| 152 | /// |
| 153 | /// This methods support demangling rules for the following <isa>: |
| 154 | /// |
| 155 | /// * AArch64: https://developer.arm.com/docs/101129/latest |
| 156 | /// |
| 157 | /// * x86 (libmvec): https://sourceware.org/glibc/wiki/libmvec and |
| 158 | /// https://sourceware.org/glibc/wiki/libmvec?action=AttachFile&do=view&target=VectorABI.txt |
| 159 | /// |
| 160 | /// \param MangledName -> input string in the format |
| 161 | /// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)]. |
| 162 | /// \param M -> Module used to retrieve informations about the vector |
| 163 | /// function that are not possible to retrieve from the mangled |
| 164 | /// name. At the moment, this parameter is needed only to retrieve the |
| 165 | /// Vectorization Factor of scalable vector functions from their |
| 166 | /// respective IR declarations. |
| 167 | Optional<VFInfo> tryDemangleForVFABI(StringRef MangledName, const Module &M); |
| 168 | |
| 169 | /// This routine mangles the given VectorName according to the LangRef |
| 170 | /// specification for vector-function-abi-variant attribute and is specific to |
| 171 | /// the TLI mappings. It is the responsibility of the caller to make sure that |
| 172 | /// this is only used if all parameters in the vector function are vector type. |
| 173 | /// This returned string holds scalar-to-vector mapping: |
| 174 | /// _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>) |
| 175 | /// |
| 176 | /// where: |
| 177 | /// |
| 178 | /// <isa> = "_LLVM_" |
| 179 | /// <mask> = "N". Note: TLI does not support masked interfaces. |
| 180 | /// <vlen> = Number of concurrent lanes, stored in the `VectorizationFactor` |
| 181 | /// field of the `VecDesc` struct. If the number of lanes is scalable |
| 182 | /// then 'x' is printed instead. |
| 183 | /// <vparams> = "v", as many as are the numArgs. |
| 184 | /// <scalarname> = the name of the scalar function. |
| 185 | /// <vectorname> = the name of the vector function. |
| 186 | std::string mangleTLIVectorName(StringRef VectorName, StringRef ScalarName, |
| 187 | unsigned numArgs, ElementCount VF); |
| 188 | |
| 189 | /// Retrieve the `VFParamKind` from a string token. |
| 190 | VFParamKind getVFParamKindFromString(const StringRef Token); |
| 191 | |
| 192 | // Name of the attribute where the variant mappings are stored. |
| 193 | static constexpr char const *MappingsAttrName = "vector-function-abi-variant"; |
| 194 | |
| 195 | /// Populates a set of strings representing the Vector Function ABI variants |
| 196 | /// associated to the CallInst CI. If the CI does not contain the |
| 197 | /// vector-function-abi-variant attribute, we return without populating |
| 198 | /// VariantMappings, i.e. callers of getVectorVariantNames need not check for |
| 199 | /// the presence of the attribute (see InjectTLIMappings). |
| 200 | void getVectorVariantNames(const CallInst &CI, |
| 201 | SmallVectorImpl<std::string> &VariantMappings); |
| 202 | } // end namespace VFABI |
| 203 | |
| 204 | /// The Vector Function Database. |
| 205 | /// |
| 206 | /// Helper class used to find the vector functions associated to a |
| 207 | /// scalar CallInst. |
| 208 | class VFDatabase { |
| 209 | /// The Module of the CallInst CI. |
| 210 | const Module *M; |
| 211 | /// The CallInst instance being queried for scalar to vector mappings. |
| 212 | const CallInst &CI; |
| 213 | /// List of vector functions descriptors associated to the call |
| 214 | /// instruction. |
| 215 | const SmallVector<VFInfo, 8> ScalarToVectorMappings; |
| 216 | |
| 217 | /// Retrieve the scalar-to-vector mappings associated to the rule of |
| 218 | /// a vector Function ABI. |
| 219 | static void getVFABIMappings(const CallInst &CI, |
| 220 | SmallVectorImpl<VFInfo> &Mappings) { |
| 221 | if (!CI.getCalledFunction()) |
| 222 | return; |
| 223 | |
| 224 | const StringRef ScalarName = CI.getCalledFunction()->getName(); |
| 225 | |
| 226 | SmallVector<std::string, 8> ListOfStrings; |
| 227 | // The check for the vector-function-abi-variant attribute is done when |
| 228 | // retrieving the vector variant names here. |
| 229 | VFABI::getVectorVariantNames(CI, ListOfStrings); |
| 230 | if (ListOfStrings.empty()) |
| 231 | return; |
| 232 | for (const auto &MangledName : ListOfStrings) { |
| 233 | const Optional<VFInfo> Shape = |
| 234 | VFABI::tryDemangleForVFABI(MangledName, *(CI.getModule())); |
| 235 | // A match is found via scalar and vector names, and also by |
| 236 | // ensuring that the variant described in the attribute has a |
| 237 | // corresponding definition or declaration of the vector |
| 238 | // function in the Module M. |
| 239 | if (Shape.hasValue() && (Shape.getValue().ScalarName == ScalarName)) { |
| 240 | assert(CI.getModule()->getFunction(Shape.getValue().VectorName) &&(static_cast <bool> (CI.getModule()->getFunction(Shape .getValue().VectorName) && "Vector function is missing." ) ? void (0) : __assert_fail ("CI.getModule()->getFunction(Shape.getValue().VectorName) && \"Vector function is missing.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 241, __extension__ __PRETTY_FUNCTION__)) |
| 241 | "Vector function is missing.")(static_cast <bool> (CI.getModule()->getFunction(Shape .getValue().VectorName) && "Vector function is missing." ) ? void (0) : __assert_fail ("CI.getModule()->getFunction(Shape.getValue().VectorName) && \"Vector function is missing.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 241, __extension__ __PRETTY_FUNCTION__)); |
| 242 | Mappings.push_back(Shape.getValue()); |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | public: |
| 248 | /// Retrieve all the VFInfo instances associated to the CallInst CI. |
| 249 | static SmallVector<VFInfo, 8> getMappings(const CallInst &CI) { |
| 250 | SmallVector<VFInfo, 8> Ret; |
| 251 | |
| 252 | // Get mappings from the Vector Function ABI variants. |
| 253 | getVFABIMappings(CI, Ret); |
| 254 | |
| 255 | // Other non-VFABI variants should be retrieved here. |
| 256 | |
| 257 | return Ret; |
| 258 | } |
| 259 | |
| 260 | /// Constructor, requires a CallInst instance. |
| 261 | VFDatabase(CallInst &CI) |
| 262 | : M(CI.getModule()), CI(CI), |
| 263 | ScalarToVectorMappings(VFDatabase::getMappings(CI)) {} |
| 264 | /// \defgroup VFDatabase query interface. |
| 265 | /// |
| 266 | /// @{ |
| 267 | /// Retrieve the Function with VFShape \p Shape. |
| 268 | Function *getVectorizedFunction(const VFShape &Shape) const { |
| 269 | if (Shape == VFShape::getScalarShape(CI)) |
| 270 | return CI.getCalledFunction(); |
| 271 | |
| 272 | for (const auto &Info : ScalarToVectorMappings) |
| 273 | if (Info.Shape == Shape) |
| 274 | return M->getFunction(Info.VectorName); |
| 275 | |
| 276 | return nullptr; |
| 277 | } |
| 278 | /// @} |
| 279 | }; |
| 280 | |
| 281 | template <typename T> class ArrayRef; |
| 282 | class DemandedBits; |
| 283 | class GetElementPtrInst; |
| 284 | template <typename InstTy> class InterleaveGroup; |
| 285 | class IRBuilderBase; |
| 286 | class Loop; |
| 287 | class ScalarEvolution; |
| 288 | class TargetTransformInfo; |
| 289 | class Type; |
| 290 | class Value; |
| 291 | |
| 292 | namespace Intrinsic { |
| 293 | typedef unsigned ID; |
| 294 | } |
| 295 | |
| 296 | /// A helper function for converting Scalar types to vector types. If |
| 297 | /// the incoming type is void, we return void. If the EC represents a |
| 298 | /// scalar, we return the scalar type. |
| 299 | inline Type *ToVectorTy(Type *Scalar, ElementCount EC) { |
| 300 | if (Scalar->isVoidTy() || Scalar->isMetadataTy() || EC.isScalar()) |
| 301 | return Scalar; |
| 302 | return VectorType::get(Scalar, EC); |
| 303 | } |
| 304 | |
| 305 | inline Type *ToVectorTy(Type *Scalar, unsigned VF) { |
| 306 | return ToVectorTy(Scalar, ElementCount::getFixed(VF)); |
| 307 | } |
| 308 | |
| 309 | /// Identify if the intrinsic is trivially vectorizable. |
| 310 | /// This method returns true if the intrinsic's argument types are all scalars |
| 311 | /// for the scalar form of the intrinsic and all vectors (or scalars handled by |
| 312 | /// hasVectorInstrinsicScalarOpd) for the vector form of the intrinsic. |
| 313 | bool isTriviallyVectorizable(Intrinsic::ID ID); |
| 314 | |
| 315 | /// Identifies if the vector form of the intrinsic has a scalar operand. |
| 316 | bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx); |
| 317 | |
| 318 | /// Identifies if the vector form of the intrinsic has a scalar operand that has |
| 319 | /// an overloaded type. |
| 320 | bool hasVectorInstrinsicOverloadedScalarOpd(Intrinsic::ID ID, |
| 321 | unsigned ScalarOpdIdx); |
| 322 | |
| 323 | /// Returns intrinsic ID for call. |
| 324 | /// For the input call instruction it finds mapping intrinsic and returns |
| 325 | /// its intrinsic ID, in case it does not found it return not_intrinsic. |
| 326 | Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, |
| 327 | const TargetLibraryInfo *TLI); |
| 328 | |
| 329 | /// Find the operand of the GEP that should be checked for consecutive |
| 330 | /// stores. This ignores trailing indices that have no effect on the final |
| 331 | /// pointer. |
| 332 | unsigned getGEPInductionOperand(const GetElementPtrInst *Gep); |
| 333 | |
| 334 | /// If the argument is a GEP, then returns the operand identified by |
| 335 | /// getGEPInductionOperand. However, if there is some other non-loop-invariant |
| 336 | /// operand, it returns that instead. |
| 337 | Value *stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp); |
| 338 | |
| 339 | /// If a value has only one user that is a CastInst, return it. |
| 340 | Value *getUniqueCastUse(Value *Ptr, Loop *Lp, Type *Ty); |
| 341 | |
| 342 | /// Get the stride of a pointer access in a loop. Looks for symbolic |
| 343 | /// strides "a[i*stride]". Returns the symbolic stride, or null otherwise. |
| 344 | Value *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp); |
| 345 | |
| 346 | /// Given a vector and an element number, see if the scalar value is |
| 347 | /// already around as a register, for example if it were inserted then extracted |
| 348 | /// from the vector. |
| 349 | Value *findScalarElement(Value *V, unsigned EltNo); |
| 350 | |
| 351 | /// If all non-negative \p Mask elements are the same value, return that value. |
| 352 | /// If all elements are negative (undefined) or \p Mask contains different |
| 353 | /// non-negative values, return -1. |
| 354 | int getSplatIndex(ArrayRef<int> Mask); |
| 355 | |
| 356 | /// Get splat value if the input is a splat vector or return nullptr. |
| 357 | /// The value may be extracted from a splat constants vector or from |
| 358 | /// a sequence of instructions that broadcast a single value into a vector. |
| 359 | Value *getSplatValue(const Value *V); |
| 360 | |
| 361 | /// Return true if each element of the vector value \p V is poisoned or equal to |
| 362 | /// every other non-poisoned element. If an index element is specified, either |
| 363 | /// every element of the vector is poisoned or the element at that index is not |
| 364 | /// poisoned and equal to every other non-poisoned element. |
| 365 | /// This may be more powerful than the related getSplatValue() because it is |
| 366 | /// not limited by finding a scalar source value to a splatted vector. |
| 367 | bool isSplatValue(const Value *V, int Index = -1, unsigned Depth = 0); |
| 368 | |
| 369 | /// Replace each shuffle mask index with the scaled sequential indices for an |
| 370 | /// equivalent mask of narrowed elements. Mask elements that are less than 0 |
| 371 | /// (sentinel values) are repeated in the output mask. |
| 372 | /// |
| 373 | /// Example with Scale = 4: |
| 374 | /// <4 x i32> <3, 2, 0, -1> --> |
| 375 | /// <16 x i8> <12, 13, 14, 15, 8, 9, 10, 11, 0, 1, 2, 3, -1, -1, -1, -1> |
| 376 | /// |
| 377 | /// This is the reverse process of widening shuffle mask elements, but it always |
| 378 | /// succeeds because the indexes can always be multiplied (scaled up) to map to |
| 379 | /// narrower vector elements. |
| 380 | void narrowShuffleMaskElts(int Scale, ArrayRef<int> Mask, |
| 381 | SmallVectorImpl<int> &ScaledMask); |
| 382 | |
| 383 | /// Try to transform a shuffle mask by replacing elements with the scaled index |
| 384 | /// for an equivalent mask of widened elements. If all mask elements that would |
| 385 | /// map to a wider element of the new mask are the same negative number |
| 386 | /// (sentinel value), that element of the new mask is the same value. If any |
| 387 | /// element in a given slice is negative and some other element in that slice is |
| 388 | /// not the same value, return false (partial matches with sentinel values are |
| 389 | /// not allowed). |
| 390 | /// |
| 391 | /// Example with Scale = 4: |
| 392 | /// <16 x i8> <12, 13, 14, 15, 8, 9, 10, 11, 0, 1, 2, 3, -1, -1, -1, -1> --> |
| 393 | /// <4 x i32> <3, 2, 0, -1> |
| 394 | /// |
| 395 | /// This is the reverse process of narrowing shuffle mask elements if it |
| 396 | /// succeeds. This transform is not always possible because indexes may not |
| 397 | /// divide evenly (scale down) to map to wider vector elements. |
| 398 | bool widenShuffleMaskElts(int Scale, ArrayRef<int> Mask, |
| 399 | SmallVectorImpl<int> &ScaledMask); |
| 400 | |
| 401 | /// Compute a map of integer instructions to their minimum legal type |
| 402 | /// size. |
| 403 | /// |
| 404 | /// C semantics force sub-int-sized values (e.g. i8, i16) to be promoted to int |
| 405 | /// type (e.g. i32) whenever arithmetic is performed on them. |
| 406 | /// |
| 407 | /// For targets with native i8 or i16 operations, usually InstCombine can shrink |
| 408 | /// the arithmetic type down again. However InstCombine refuses to create |
| 409 | /// illegal types, so for targets without i8 or i16 registers, the lengthening |
| 410 | /// and shrinking remains. |
| 411 | /// |
| 412 | /// Most SIMD ISAs (e.g. NEON) however support vectors of i8 or i16 even when |
| 413 | /// their scalar equivalents do not, so during vectorization it is important to |
| 414 | /// remove these lengthens and truncates when deciding the profitability of |
| 415 | /// vectorization. |
| 416 | /// |
| 417 | /// This function analyzes the given range of instructions and determines the |
| 418 | /// minimum type size each can be converted to. It attempts to remove or |
| 419 | /// minimize type size changes across each def-use chain, so for example in the |
| 420 | /// following code: |
| 421 | /// |
| 422 | /// %1 = load i8, i8* |
| 423 | /// %2 = add i8 %1, 2 |
| 424 | /// %3 = load i16, i16* |
| 425 | /// %4 = zext i8 %2 to i32 |
| 426 | /// %5 = zext i16 %3 to i32 |
| 427 | /// %6 = add i32 %4, %5 |
| 428 | /// %7 = trunc i32 %6 to i16 |
| 429 | /// |
| 430 | /// Instruction %6 must be done at least in i16, so computeMinimumValueSizes |
| 431 | /// will return: {%1: 16, %2: 16, %3: 16, %4: 16, %5: 16, %6: 16, %7: 16}. |
| 432 | /// |
| 433 | /// If the optional TargetTransformInfo is provided, this function tries harder |
| 434 | /// to do less work by only looking at illegal types. |
| 435 | MapVector<Instruction*, uint64_t> |
| 436 | computeMinimumValueSizes(ArrayRef<BasicBlock*> Blocks, |
| 437 | DemandedBits &DB, |
| 438 | const TargetTransformInfo *TTI=nullptr); |
| 439 | |
| 440 | /// Compute the union of two access-group lists. |
| 441 | /// |
| 442 | /// If the list contains just one access group, it is returned directly. If the |
| 443 | /// list is empty, returns nullptr. |
| 444 | MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2); |
| 445 | |
| 446 | /// Compute the access-group list of access groups that @p Inst1 and @p Inst2 |
| 447 | /// are both in. If either instruction does not access memory at all, it is |
| 448 | /// considered to be in every list. |
| 449 | /// |
| 450 | /// If the list contains just one access group, it is returned directly. If the |
| 451 | /// list is empty, returns nullptr. |
| 452 | MDNode *intersectAccessGroups(const Instruction *Inst1, |
| 453 | const Instruction *Inst2); |
| 454 | |
| 455 | /// Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, |
| 456 | /// MD_nontemporal, MD_access_group]. |
| 457 | /// For K in Kinds, we get the MDNode for K from each of the |
| 458 | /// elements of VL, compute their "intersection" (i.e., the most generic |
| 459 | /// metadata value that covers all of the individual values), and set I's |
| 460 | /// metadata for M equal to the intersection value. |
| 461 | /// |
| 462 | /// This function always sets a (possibly null) value for each K in Kinds. |
| 463 | Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL); |
| 464 | |
| 465 | /// Create a mask that filters the members of an interleave group where there |
| 466 | /// are gaps. |
| 467 | /// |
| 468 | /// For example, the mask for \p Group with interleave-factor 3 |
| 469 | /// and \p VF 4, that has only its first member present is: |
| 470 | /// |
| 471 | /// <1,0,0,1,0,0,1,0,0,1,0,0> |
| 472 | /// |
| 473 | /// Note: The result is a mask of 0's and 1's, as opposed to the other |
| 474 | /// create[*]Mask() utilities which create a shuffle mask (mask that |
| 475 | /// consists of indices). |
| 476 | Constant *createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, |
| 477 | const InterleaveGroup<Instruction> &Group); |
| 478 | |
| 479 | /// Create a mask with replicated elements. |
| 480 | /// |
| 481 | /// This function creates a shuffle mask for replicating each of the \p VF |
| 482 | /// elements in a vector \p ReplicationFactor times. It can be used to |
| 483 | /// transform a mask of \p VF elements into a mask of |
| 484 | /// \p VF * \p ReplicationFactor elements used by a predicated |
| 485 | /// interleaved-group of loads/stores whose Interleaved-factor == |
| 486 | /// \p ReplicationFactor. |
| 487 | /// |
| 488 | /// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is: |
| 489 | /// |
| 490 | /// <0,0,0,1,1,1,2,2,2,3,3,3> |
| 491 | llvm::SmallVector<int, 16> createReplicatedMask(unsigned ReplicationFactor, |
| 492 | unsigned VF); |
| 493 | |
| 494 | /// Create an interleave shuffle mask. |
| 495 | /// |
| 496 | /// This function creates a shuffle mask for interleaving \p NumVecs vectors of |
| 497 | /// vectorization factor \p VF into a single wide vector. The mask is of the |
| 498 | /// form: |
| 499 | /// |
| 500 | /// <0, VF, VF * 2, ..., VF * (NumVecs - 1), 1, VF + 1, VF * 2 + 1, ...> |
| 501 | /// |
| 502 | /// For example, the mask for VF = 4 and NumVecs = 2 is: |
| 503 | /// |
| 504 | /// <0, 4, 1, 5, 2, 6, 3, 7>. |
| 505 | llvm::SmallVector<int, 16> createInterleaveMask(unsigned VF, unsigned NumVecs); |
| 506 | |
| 507 | /// Create a stride shuffle mask. |
| 508 | /// |
| 509 | /// This function creates a shuffle mask whose elements begin at \p Start and |
| 510 | /// are incremented by \p Stride. The mask can be used to deinterleave an |
| 511 | /// interleaved vector into separate vectors of vectorization factor \p VF. The |
| 512 | /// mask is of the form: |
| 513 | /// |
| 514 | /// <Start, Start + Stride, ..., Start + Stride * (VF - 1)> |
| 515 | /// |
| 516 | /// For example, the mask for Start = 0, Stride = 2, and VF = 4 is: |
| 517 | /// |
| 518 | /// <0, 2, 4, 6> |
| 519 | llvm::SmallVector<int, 16> createStrideMask(unsigned Start, unsigned Stride, |
| 520 | unsigned VF); |
| 521 | |
| 522 | /// Create a sequential shuffle mask. |
| 523 | /// |
| 524 | /// This function creates shuffle mask whose elements are sequential and begin |
| 525 | /// at \p Start. The mask contains \p NumInts integers and is padded with \p |
| 526 | /// NumUndefs undef values. The mask is of the form: |
| 527 | /// |
| 528 | /// <Start, Start + 1, ... Start + NumInts - 1, undef_1, ... undef_NumUndefs> |
| 529 | /// |
| 530 | /// For example, the mask for Start = 0, NumInsts = 4, and NumUndefs = 4 is: |
| 531 | /// |
| 532 | /// <0, 1, 2, 3, undef, undef, undef, undef> |
| 533 | llvm::SmallVector<int, 16> |
| 534 | createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs); |
| 535 | |
| 536 | /// Concatenate a list of vectors. |
| 537 | /// |
| 538 | /// This function generates code that concatenate the vectors in \p Vecs into a |
| 539 | /// single large vector. The number of vectors should be greater than one, and |
| 540 | /// their element types should be the same. The number of elements in the |
| 541 | /// vectors should also be the same; however, if the last vector has fewer |
| 542 | /// elements, it will be padded with undefs. |
| 543 | Value *concatenateVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vecs); |
| 544 | |
| 545 | /// Given a mask vector of i1, Return true if all of the elements of this |
| 546 | /// predicate mask are known to be false or undef. That is, return true if all |
| 547 | /// lanes can be assumed inactive. |
| 548 | bool maskIsAllZeroOrUndef(Value *Mask); |
| 549 | |
| 550 | /// Given a mask vector of i1, Return true if all of the elements of this |
| 551 | /// predicate mask are known to be true or undef. That is, return true if all |
| 552 | /// lanes can be assumed active. |
| 553 | bool maskIsAllOneOrUndef(Value *Mask); |
| 554 | |
| 555 | /// Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y) |
| 556 | /// for each lane which may be active. |
| 557 | APInt possiblyDemandedEltsInMask(Value *Mask); |
| 558 | |
| 559 | /// The group of interleaved loads/stores sharing the same stride and |
| 560 | /// close to each other. |
| 561 | /// |
| 562 | /// Each member in this group has an index starting from 0, and the largest |
| 563 | /// index should be less than interleaved factor, which is equal to the absolute |
| 564 | /// value of the access's stride. |
| 565 | /// |
| 566 | /// E.g. An interleaved load group of factor 4: |
| 567 | /// for (unsigned i = 0; i < 1024; i+=4) { |
| 568 | /// a = A[i]; // Member of index 0 |
| 569 | /// b = A[i+1]; // Member of index 1 |
| 570 | /// d = A[i+3]; // Member of index 3 |
| 571 | /// ... |
| 572 | /// } |
| 573 | /// |
| 574 | /// An interleaved store group of factor 4: |
| 575 | /// for (unsigned i = 0; i < 1024; i+=4) { |
| 576 | /// ... |
| 577 | /// A[i] = a; // Member of index 0 |
| 578 | /// A[i+1] = b; // Member of index 1 |
| 579 | /// A[i+2] = c; // Member of index 2 |
| 580 | /// A[i+3] = d; // Member of index 3 |
| 581 | /// } |
| 582 | /// |
| 583 | /// Note: the interleaved load group could have gaps (missing members), but |
| 584 | /// the interleaved store group doesn't allow gaps. |
| 585 | template <typename InstTy> class InterleaveGroup { |
| 586 | public: |
| 587 | InterleaveGroup(uint32_t Factor, bool Reverse, Align Alignment) |
| 588 | : Factor(Factor), Reverse(Reverse), Alignment(Alignment), |
| 589 | InsertPos(nullptr) {} |
| 590 | |
| 591 | InterleaveGroup(InstTy *Instr, int32_t Stride, Align Alignment) |
| 592 | : Alignment(Alignment), InsertPos(Instr) { |
| 593 | Factor = std::abs(Stride); |
| 594 | assert(Factor > 1 && "Invalid interleave factor")(static_cast <bool> (Factor > 1 && "Invalid interleave factor" ) ? void (0) : __assert_fail ("Factor > 1 && \"Invalid interleave factor\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 594, __extension__ __PRETTY_FUNCTION__)); |
| 595 | |
| 596 | Reverse = Stride < 0; |
| 597 | Members[0] = Instr; |
| 598 | } |
| 599 | |
| 600 | bool isReverse() const { return Reverse; } |
| 601 | uint32_t getFactor() const { return Factor; } |
| 602 | Align getAlign() const { return Alignment; } |
| 603 | uint32_t getNumMembers() const { return Members.size(); } |
| 604 | |
| 605 | /// Try to insert a new member \p Instr with index \p Index and |
| 606 | /// alignment \p NewAlign. The index is related to the leader and it could be |
| 607 | /// negative if it is the new leader. |
| 608 | /// |
| 609 | /// \returns false if the instruction doesn't belong to the group. |
| 610 | bool insertMember(InstTy *Instr, int32_t Index, Align NewAlign) { |
| 611 | // Make sure the key fits in an int32_t. |
| 612 | Optional<int32_t> MaybeKey = checkedAdd(Index, SmallestKey); |
| 613 | if (!MaybeKey) |
| 614 | return false; |
| 615 | int32_t Key = *MaybeKey; |
| 616 | |
| 617 | // Skip if the key is used for either the tombstone or empty special values. |
| 618 | if (DenseMapInfo<int32_t>::getTombstoneKey() == Key || |
| 619 | DenseMapInfo<int32_t>::getEmptyKey() == Key) |
| 620 | return false; |
| 621 | |
| 622 | // Skip if there is already a member with the same index. |
| 623 | if (Members.find(Key) != Members.end()) |
| 624 | return false; |
| 625 | |
| 626 | if (Key > LargestKey) { |
| 627 | // The largest index is always less than the interleave factor. |
| 628 | if (Index >= static_cast<int32_t>(Factor)) |
| 629 | return false; |
| 630 | |
| 631 | LargestKey = Key; |
| 632 | } else if (Key < SmallestKey) { |
| 633 | |
| 634 | // Make sure the largest index fits in an int32_t. |
| 635 | Optional<int32_t> MaybeLargestIndex = checkedSub(LargestKey, Key); |
| 636 | if (!MaybeLargestIndex) |
| 637 | return false; |
| 638 | |
| 639 | // The largest index is always less than the interleave factor. |
| 640 | if (*MaybeLargestIndex >= static_cast<int64_t>(Factor)) |
| 641 | return false; |
| 642 | |
| 643 | SmallestKey = Key; |
| 644 | } |
| 645 | |
| 646 | // It's always safe to select the minimum alignment. |
| 647 | Alignment = std::min(Alignment, NewAlign); |
| 648 | Members[Key] = Instr; |
| 649 | return true; |
| 650 | } |
| 651 | |
| 652 | /// Get the member with the given index \p Index |
| 653 | /// |
| 654 | /// \returns nullptr if contains no such member. |
| 655 | InstTy *getMember(uint32_t Index) const { |
| 656 | int32_t Key = SmallestKey + Index; |
| 657 | return Members.lookup(Key); |
| 658 | } |
| 659 | |
| 660 | /// Get the index for the given member. Unlike the key in the member |
| 661 | /// map, the index starts from 0. |
| 662 | uint32_t getIndex(const InstTy *Instr) const { |
| 663 | for (auto I : Members) { |
| 664 | if (I.second == Instr) |
| 665 | return I.first - SmallestKey; |
| 666 | } |
| 667 | |
| 668 | llvm_unreachable("InterleaveGroup contains no such member")::llvm::llvm_unreachable_internal("InterleaveGroup contains no such member" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 668); |
| 669 | } |
| 670 | |
| 671 | InstTy *getInsertPos() const { return InsertPos; } |
| 672 | void setInsertPos(InstTy *Inst) { InsertPos = Inst; } |
| 673 | |
| 674 | /// Add metadata (e.g. alias info) from the instructions in this group to \p |
| 675 | /// NewInst. |
| 676 | /// |
| 677 | /// FIXME: this function currently does not add noalias metadata a'la |
| 678 | /// addNewMedata. To do that we need to compute the intersection of the |
| 679 | /// noalias info from all members. |
| 680 | void addMetadata(InstTy *NewInst) const; |
| 681 | |
| 682 | /// Returns true if this Group requires a scalar iteration to handle gaps. |
| 683 | bool requiresScalarEpilogue() const { |
| 684 | // If the last member of the Group exists, then a scalar epilog is not |
| 685 | // needed for this group. |
| 686 | if (getMember(getFactor() - 1)) |
| 687 | return false; |
| 688 | |
| 689 | // We have a group with gaps. It therefore cannot be a group of stores, |
| 690 | // and it can't be a reversed access, because such groups get invalidated. |
| 691 | assert(!getMember(0)->mayWriteToMemory() &&(static_cast <bool> (!getMember(0)->mayWriteToMemory () && "Group should have been invalidated") ? void (0 ) : __assert_fail ("!getMember(0)->mayWriteToMemory() && \"Group should have been invalidated\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 692, __extension__ __PRETTY_FUNCTION__)) |
| 692 | "Group should have been invalidated")(static_cast <bool> (!getMember(0)->mayWriteToMemory () && "Group should have been invalidated") ? void (0 ) : __assert_fail ("!getMember(0)->mayWriteToMemory() && \"Group should have been invalidated\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 692, __extension__ __PRETTY_FUNCTION__)); |
| 693 | assert(!isReverse() && "Group should have been invalidated")(static_cast <bool> (!isReverse() && "Group should have been invalidated" ) ? void (0) : __assert_fail ("!isReverse() && \"Group should have been invalidated\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 693, __extension__ __PRETTY_FUNCTION__)); |
| 694 | |
| 695 | // This is a group of loads, with gaps, and without a last-member |
| 696 | return true; |
| 697 | } |
| 698 | |
| 699 | private: |
| 700 | uint32_t Factor; // Interleave Factor. |
| 701 | bool Reverse; |
| 702 | Align Alignment; |
| 703 | DenseMap<int32_t, InstTy *> Members; |
| 704 | int32_t SmallestKey = 0; |
| 705 | int32_t LargestKey = 0; |
| 706 | |
| 707 | // To avoid breaking dependences, vectorized instructions of an interleave |
| 708 | // group should be inserted at either the first load or the last store in |
| 709 | // program order. |
| 710 | // |
| 711 | // E.g. %even = load i32 // Insert Position |
| 712 | // %add = add i32 %even // Use of %even |
| 713 | // %odd = load i32 |
| 714 | // |
| 715 | // store i32 %even |
| 716 | // %odd = add i32 // Def of %odd |
| 717 | // store i32 %odd // Insert Position |
| 718 | InstTy *InsertPos; |
| 719 | }; |
| 720 | |
| 721 | /// Drive the analysis of interleaved memory accesses in the loop. |
| 722 | /// |
| 723 | /// Use this class to analyze interleaved accesses only when we can vectorize |
| 724 | /// a loop. Otherwise it's meaningless to do analysis as the vectorization |
| 725 | /// on interleaved accesses is unsafe. |
| 726 | /// |
| 727 | /// The analysis collects interleave groups and records the relationships |
| 728 | /// between the member and the group in a map. |
| 729 | class InterleavedAccessInfo { |
| 730 | public: |
| 731 | InterleavedAccessInfo(PredicatedScalarEvolution &PSE, Loop *L, |
| 732 | DominatorTree *DT, LoopInfo *LI, |
| 733 | const LoopAccessInfo *LAI) |
| 734 | : PSE(PSE), TheLoop(L), DT(DT), LI(LI), LAI(LAI) {} |
| 735 | |
| 736 | ~InterleavedAccessInfo() { invalidateGroups(); } |
| 737 | |
| 738 | /// Analyze the interleaved accesses and collect them in interleave |
| 739 | /// groups. Substitute symbolic strides using \p Strides. |
| 740 | /// Consider also predicated loads/stores in the analysis if |
| 741 | /// \p EnableMaskedInterleavedGroup is true. |
| 742 | void analyzeInterleaving(bool EnableMaskedInterleavedGroup); |
| 743 | |
| 744 | /// Invalidate groups, e.g., in case all blocks in loop will be predicated |
| 745 | /// contrary to original assumption. Although we currently prevent group |
| 746 | /// formation for predicated accesses, we may be able to relax this limitation |
| 747 | /// in the future once we handle more complicated blocks. Returns true if any |
| 748 | /// groups were invalidated. |
| 749 | bool invalidateGroups() { |
| 750 | if (InterleaveGroups.empty()) { |
| 751 | assert((static_cast <bool> (!RequiresScalarEpilogue && "RequiresScalarEpilog should not be set without interleave groups" ) ? void (0) : __assert_fail ("!RequiresScalarEpilogue && \"RequiresScalarEpilog should not be set without interleave groups\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 753, __extension__ __PRETTY_FUNCTION__)) |
| 752 | !RequiresScalarEpilogue &&(static_cast <bool> (!RequiresScalarEpilogue && "RequiresScalarEpilog should not be set without interleave groups" ) ? void (0) : __assert_fail ("!RequiresScalarEpilogue && \"RequiresScalarEpilog should not be set without interleave groups\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 753, __extension__ __PRETTY_FUNCTION__)) |
| 753 | "RequiresScalarEpilog should not be set without interleave groups")(static_cast <bool> (!RequiresScalarEpilogue && "RequiresScalarEpilog should not be set without interleave groups" ) ? void (0) : __assert_fail ("!RequiresScalarEpilogue && \"RequiresScalarEpilog should not be set without interleave groups\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 753, __extension__ __PRETTY_FUNCTION__)); |
| 754 | return false; |
| 755 | } |
| 756 | |
| 757 | InterleaveGroupMap.clear(); |
| 758 | for (auto *Ptr : InterleaveGroups) |
| 759 | delete Ptr; |
| 760 | InterleaveGroups.clear(); |
| 761 | RequiresScalarEpilogue = false; |
| 762 | return true; |
| 763 | } |
| 764 | |
| 765 | /// Check if \p Instr belongs to any interleave group. |
| 766 | bool isInterleaved(Instruction *Instr) const { |
| 767 | return InterleaveGroupMap.find(Instr) != InterleaveGroupMap.end(); |
| 768 | } |
| 769 | |
| 770 | /// Get the interleave group that \p Instr belongs to. |
| 771 | /// |
| 772 | /// \returns nullptr if doesn't have such group. |
| 773 | InterleaveGroup<Instruction> * |
| 774 | getInterleaveGroup(const Instruction *Instr) const { |
| 775 | return InterleaveGroupMap.lookup(Instr); |
| 776 | } |
| 777 | |
| 778 | iterator_range<SmallPtrSetIterator<llvm::InterleaveGroup<Instruction> *>> |
| 779 | getInterleaveGroups() { |
| 780 | return make_range(InterleaveGroups.begin(), InterleaveGroups.end()); |
| 781 | } |
| 782 | |
| 783 | /// Returns true if an interleaved group that may access memory |
| 784 | /// out-of-bounds requires a scalar epilogue iteration for correctness. |
| 785 | bool requiresScalarEpilogue() const { return RequiresScalarEpilogue; } |
| 786 | |
| 787 | /// Invalidate groups that require a scalar epilogue (due to gaps). This can |
| 788 | /// happen when optimizing for size forbids a scalar epilogue, and the gap |
| 789 | /// cannot be filtered by masking the load/store. |
| 790 | void invalidateGroupsRequiringScalarEpilogue(); |
| 791 | |
| 792 | private: |
| 793 | /// A wrapper around ScalarEvolution, used to add runtime SCEV checks. |
| 794 | /// Simplifies SCEV expressions in the context of existing SCEV assumptions. |
| 795 | /// The interleaved access analysis can also add new predicates (for example |
| 796 | /// by versioning strides of pointers). |
| 797 | PredicatedScalarEvolution &PSE; |
| 798 | |
| 799 | Loop *TheLoop; |
| 800 | DominatorTree *DT; |
| 801 | LoopInfo *LI; |
| 802 | const LoopAccessInfo *LAI; |
| 803 | |
| 804 | /// True if the loop may contain non-reversed interleaved groups with |
| 805 | /// out-of-bounds accesses. We ensure we don't speculatively access memory |
| 806 | /// out-of-bounds by executing at least one scalar epilogue iteration. |
| 807 | bool RequiresScalarEpilogue = false; |
| 808 | |
| 809 | /// Holds the relationships between the members and the interleave group. |
| 810 | DenseMap<Instruction *, InterleaveGroup<Instruction> *> InterleaveGroupMap; |
| 811 | |
| 812 | SmallPtrSet<InterleaveGroup<Instruction> *, 4> InterleaveGroups; |
| 813 | |
| 814 | /// Holds dependences among the memory accesses in the loop. It maps a source |
| 815 | /// access to a set of dependent sink accesses. |
| 816 | DenseMap<Instruction *, SmallPtrSet<Instruction *, 2>> Dependences; |
| 817 | |
| 818 | /// The descriptor for a strided memory access. |
| 819 | struct StrideDescriptor { |
| 820 | StrideDescriptor() = default; |
| 821 | StrideDescriptor(int64_t Stride, const SCEV *Scev, uint64_t Size, |
| 822 | Align Alignment) |
| 823 | : Stride(Stride), Scev(Scev), Size(Size), Alignment(Alignment) {} |
| 824 | |
| 825 | // The access's stride. It is negative for a reverse access. |
| 826 | int64_t Stride = 0; |
| 827 | |
| 828 | // The scalar expression of this access. |
| 829 | const SCEV *Scev = nullptr; |
| 830 | |
| 831 | // The size of the memory object. |
| 832 | uint64_t Size = 0; |
| 833 | |
| 834 | // The alignment of this access. |
| 835 | Align Alignment; |
| 836 | }; |
| 837 | |
| 838 | /// A type for holding instructions and their stride descriptors. |
| 839 | using StrideEntry = std::pair<Instruction *, StrideDescriptor>; |
| 840 | |
| 841 | /// Create a new interleave group with the given instruction \p Instr, |
| 842 | /// stride \p Stride and alignment \p Align. |
| 843 | /// |
| 844 | /// \returns the newly created interleave group. |
| 845 | InterleaveGroup<Instruction> * |
| 846 | createInterleaveGroup(Instruction *Instr, int Stride, Align Alignment) { |
| 847 | assert(!InterleaveGroupMap.count(Instr) &&(static_cast <bool> (!InterleaveGroupMap.count(Instr) && "Already in an interleaved access group") ? void (0) : __assert_fail ("!InterleaveGroupMap.count(Instr) && \"Already in an interleaved access group\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 848, __extension__ __PRETTY_FUNCTION__)) |
| 848 | "Already in an interleaved access group")(static_cast <bool> (!InterleaveGroupMap.count(Instr) && "Already in an interleaved access group") ? void (0) : __assert_fail ("!InterleaveGroupMap.count(Instr) && \"Already in an interleaved access group\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/Analysis/VectorUtils.h" , 848, __extension__ __PRETTY_FUNCTION__)); |
| 849 | InterleaveGroupMap[Instr] = |
| 850 | new InterleaveGroup<Instruction>(Instr, Stride, Alignment); |
| 851 | InterleaveGroups.insert(InterleaveGroupMap[Instr]); |
| 852 | return InterleaveGroupMap[Instr]; |
| 853 | } |
| 854 | |
| 855 | /// Release the group and remove all the relationships. |
| 856 | void releaseGroup(InterleaveGroup<Instruction> *Group) { |
| 857 | for (unsigned i = 0; i < Group->getFactor(); i++) |
| 858 | if (Instruction *Member = Group->getMember(i)) |
| 859 | InterleaveGroupMap.erase(Member); |
| 860 | |
| 861 | InterleaveGroups.erase(Group); |
| 862 | delete Group; |
| 863 | } |
| 864 | |
| 865 | /// Collect all the accesses with a constant stride in program order. |
| 866 | void collectConstStrideAccesses( |
| 867 | MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo, |
| 868 | const ValueToValueMap &Strides); |
| 869 | |
| 870 | /// Returns true if \p Stride is allowed in an interleaved group. |
| 871 | static bool isStrided(int Stride); |
| 872 | |
| 873 | /// Returns true if \p BB is a predicated block. |
| 874 | bool isPredicated(BasicBlock *BB) const { |
| 875 | return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT); |
| 876 | } |
| 877 | |
| 878 | /// Returns true if LoopAccessInfo can be used for dependence queries. |
| 879 | bool areDependencesValid() const { |
| 880 | return LAI && LAI->getDepChecker().getDependences(); |
| 881 | } |
| 882 | |
| 883 | /// Returns true if memory accesses \p A and \p B can be reordered, if |
| 884 | /// necessary, when constructing interleaved groups. |
| 885 | /// |
| 886 | /// \p A must precede \p B in program order. We return false if reordering is |
| 887 | /// not necessary or is prevented because \p A and \p B may be dependent. |
| 888 | bool canReorderMemAccessesForInterleavedGroups(StrideEntry *A, |
| 889 | StrideEntry *B) const { |
| 890 | // Code motion for interleaved accesses can potentially hoist strided loads |
| 891 | // and sink strided stores. The code below checks the legality of the |
| 892 | // following two conditions: |
| 893 | // |
| 894 | // 1. Potentially moving a strided load (B) before any store (A) that |
| 895 | // precedes B, or |
| 896 | // |
| 897 | // 2. Potentially moving a strided store (A) after any load or store (B) |
| 898 | // that A precedes. |
| 899 | // |
| 900 | // It's legal to reorder A and B if we know there isn't a dependence from A |
| 901 | // to B. Note that this determination is conservative since some |
| 902 | // dependences could potentially be reordered safely. |
| 903 | |
| 904 | // A is potentially the source of a dependence. |
| 905 | auto *Src = A->first; |
| 906 | auto SrcDes = A->second; |
| 907 | |
| 908 | // B is potentially the sink of a dependence. |
| 909 | auto *Sink = B->first; |
| 910 | auto SinkDes = B->second; |
| 911 | |
| 912 | // Code motion for interleaved accesses can't violate WAR dependences. |
| 913 | // Thus, reordering is legal if the source isn't a write. |
| 914 | if (!Src->mayWriteToMemory()) |
| 915 | return true; |
| 916 | |
| 917 | // At least one of the accesses must be strided. |
| 918 | if (!isStrided(SrcDes.Stride) && !isStrided(SinkDes.Stride)) |
| 919 | return true; |
| 920 | |
| 921 | // If dependence information is not available from LoopAccessInfo, |
| 922 | // conservatively assume the instructions can't be reordered. |
| 923 | if (!areDependencesValid()) |
| 924 | return false; |
| 925 | |
| 926 | // If we know there is a dependence from source to sink, assume the |
| 927 | // instructions can't be reordered. Otherwise, reordering is legal. |
| 928 | return Dependences.find(Src) == Dependences.end() || |
| 929 | !Dependences.lookup(Src).count(Sink); |
| 930 | } |
| 931 | |
| 932 | /// Collect the dependences from LoopAccessInfo. |
| 933 | /// |
| 934 | /// We process the dependences once during the interleaved access analysis to |
| 935 | /// enable constant-time dependence queries. |
| 936 | void collectDependences() { |
| 937 | if (!areDependencesValid()) |
| 938 | return; |
| 939 | auto *Deps = LAI->getDepChecker().getDependences(); |
| 940 | for (auto Dep : *Deps) |
| 941 | Dependences[Dep.getSource(*LAI)].insert(Dep.getDestination(*LAI)); |
| 942 | } |
| 943 | }; |
| 944 | |
| 945 | } // llvm namespace |
| 946 | |
| 947 | #endif |
| 1 | //===- llvm/ADT/DenseMap.h - Dense probed hash table ------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines the DenseMap class. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef LLVM_ADT_DENSEMAP_H |
| 14 | #define LLVM_ADT_DENSEMAP_H |
| 15 | |
| 16 | #include "llvm/ADT/DenseMapInfo.h" |
| 17 | #include "llvm/ADT/EpochTracker.h" |
| 18 | #include "llvm/Support/AlignOf.h" |
| 19 | #include "llvm/Support/Compiler.h" |
| 20 | #include "llvm/Support/MathExtras.h" |
| 21 | #include "llvm/Support/MemAlloc.h" |
| 22 | #include "llvm/Support/ReverseIteration.h" |
| 23 | #include "llvm/Support/type_traits.h" |
| 24 | #include <algorithm> |
| 25 | #include <cassert> |
| 26 | #include <cstddef> |
| 27 | #include <cstring> |
| 28 | #include <initializer_list> |
| 29 | #include <iterator> |
| 30 | #include <new> |
| 31 | #include <type_traits> |
| 32 | #include <utility> |
| 33 | |
| 34 | namespace llvm { |
| 35 | |
| 36 | namespace detail { |
| 37 | |
| 38 | // We extend a pair to allow users to override the bucket type with their own |
| 39 | // implementation without requiring two members. |
| 40 | template <typename KeyT, typename ValueT> |
| 41 | struct DenseMapPair : public std::pair<KeyT, ValueT> { |
| 42 | using std::pair<KeyT, ValueT>::pair; |
| 43 | |
| 44 | KeyT &getFirst() { return std::pair<KeyT, ValueT>::first; } |
| 45 | const KeyT &getFirst() const { return std::pair<KeyT, ValueT>::first; } |
| 46 | ValueT &getSecond() { return std::pair<KeyT, ValueT>::second; } |
| 47 | const ValueT &getSecond() const { return std::pair<KeyT, ValueT>::second; } |
| 48 | }; |
| 49 | |
| 50 | } // end namespace detail |
| 51 | |
| 52 | template <typename KeyT, typename ValueT, |
| 53 | typename KeyInfoT = DenseMapInfo<KeyT>, |
| 54 | typename Bucket = llvm::detail::DenseMapPair<KeyT, ValueT>, |
| 55 | bool IsConst = false> |
| 56 | class DenseMapIterator; |
| 57 | |
| 58 | template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT, |
| 59 | typename BucketT> |
| 60 | class DenseMapBase : public DebugEpochBase { |
| 61 | template <typename T> |
| 62 | using const_arg_type_t = typename const_pointer_or_const_ref<T>::type; |
| 63 | |
| 64 | public: |
| 65 | using size_type = unsigned; |
| 66 | using key_type = KeyT; |
| 67 | using mapped_type = ValueT; |
| 68 | using value_type = BucketT; |
| 69 | |
| 70 | using iterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT>; |
| 71 | using const_iterator = |
| 72 | DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>; |
| 73 | |
| 74 | inline iterator begin() { |
| 75 | // When the map is empty, avoid the overhead of advancing/retreating past |
| 76 | // empty buckets. |
| 77 | if (empty()) |
| 78 | return end(); |
| 79 | if (shouldReverseIterate<KeyT>()) |
| 80 | return makeIterator(getBucketsEnd() - 1, getBuckets(), *this); |
| 81 | return makeIterator(getBuckets(), getBucketsEnd(), *this); |
| 82 | } |
| 83 | inline iterator end() { |
| 84 | return makeIterator(getBucketsEnd(), getBucketsEnd(), *this, true); |
| 85 | } |
| 86 | inline const_iterator begin() const { |
| 87 | if (empty()) |
| 88 | return end(); |
| 89 | if (shouldReverseIterate<KeyT>()) |
| 90 | return makeConstIterator(getBucketsEnd() - 1, getBuckets(), *this); |
| 91 | return makeConstIterator(getBuckets(), getBucketsEnd(), *this); |
| 92 | } |
| 93 | inline const_iterator end() const { |
| 94 | return makeConstIterator(getBucketsEnd(), getBucketsEnd(), *this, true); |
| 95 | } |
| 96 | |
| 97 | LLVM_NODISCARD[[clang::warn_unused_result]] bool empty() const { |
| 98 | return getNumEntries() == 0; |
| 99 | } |
| 100 | unsigned size() const { return getNumEntries(); } |
| 101 | |
| 102 | /// Grow the densemap so that it can contain at least \p NumEntries items |
| 103 | /// before resizing again. |
| 104 | void reserve(size_type NumEntries) { |
| 105 | auto NumBuckets = getMinBucketToReserveForEntries(NumEntries); |
| 106 | incrementEpoch(); |
| 107 | if (NumBuckets > getNumBuckets()) |
| 108 | grow(NumBuckets); |
| 109 | } |
| 110 | |
| 111 | void clear() { |
| 112 | incrementEpoch(); |
| 113 | if (getNumEntries() == 0 && getNumTombstones() == 0) return; |
| 114 | |
| 115 | // If the capacity of the array is huge, and the # elements used is small, |
| 116 | // shrink the array. |
| 117 | if (getNumEntries() * 4 < getNumBuckets() && getNumBuckets() > 64) { |
| 118 | shrink_and_clear(); |
| 119 | return; |
| 120 | } |
| 121 | |
| 122 | const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); |
| 123 | if (std::is_trivially_destructible<ValueT>::value) { |
| 124 | // Use a simpler loop when values don't need destruction. |
| 125 | for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) |
| 126 | P->getFirst() = EmptyKey; |
| 127 | } else { |
| 128 | unsigned NumEntries = getNumEntries(); |
| 129 | for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { |
| 130 | if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { |
| 131 | if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { |
| 132 | P->getSecond().~ValueT(); |
| 133 | --NumEntries; |
| 134 | } |
| 135 | P->getFirst() = EmptyKey; |
| 136 | } |
| 137 | } |
| 138 | assert(NumEntries == 0 && "Node count imbalance!")(static_cast <bool> (NumEntries == 0 && "Node count imbalance!" ) ? void (0) : __assert_fail ("NumEntries == 0 && \"Node count imbalance!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 138, __extension__ __PRETTY_FUNCTION__)); |
| 139 | } |
| 140 | setNumEntries(0); |
| 141 | setNumTombstones(0); |
| 142 | } |
| 143 | |
| 144 | /// Return 1 if the specified key is in the map, 0 otherwise. |
| 145 | size_type count(const_arg_type_t<KeyT> Val) const { |
| 146 | const BucketT *TheBucket; |
| 147 | return LookupBucketFor(Val, TheBucket) ? 1 : 0; |
| 148 | } |
| 149 | |
| 150 | iterator find(const_arg_type_t<KeyT> Val) { |
| 151 | BucketT *TheBucket; |
| 152 | if (LookupBucketFor(Val, TheBucket)) |
| 153 | return makeIterator(TheBucket, |
| 154 | shouldReverseIterate<KeyT>() ? getBuckets() |
| 155 | : getBucketsEnd(), |
| 156 | *this, true); |
| 157 | return end(); |
| 158 | } |
| 159 | const_iterator find(const_arg_type_t<KeyT> Val) const { |
| 160 | const BucketT *TheBucket; |
| 161 | if (LookupBucketFor(Val, TheBucket)) |
| 162 | return makeConstIterator(TheBucket, |
| 163 | shouldReverseIterate<KeyT>() ? getBuckets() |
| 164 | : getBucketsEnd(), |
| 165 | *this, true); |
| 166 | return end(); |
| 167 | } |
| 168 | |
| 169 | /// Alternate version of find() which allows a different, and possibly |
| 170 | /// less expensive, key type. |
| 171 | /// The DenseMapInfo is responsible for supplying methods |
| 172 | /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key |
| 173 | /// type used. |
| 174 | template<class LookupKeyT> |
| 175 | iterator find_as(const LookupKeyT &Val) { |
| 176 | BucketT *TheBucket; |
| 177 | if (LookupBucketFor(Val, TheBucket)) |
| 178 | return makeIterator(TheBucket, |
| 179 | shouldReverseIterate<KeyT>() ? getBuckets() |
| 180 | : getBucketsEnd(), |
| 181 | *this, true); |
| 182 | return end(); |
| 183 | } |
| 184 | template<class LookupKeyT> |
| 185 | const_iterator find_as(const LookupKeyT &Val) const { |
| 186 | const BucketT *TheBucket; |
| 187 | if (LookupBucketFor(Val, TheBucket)) |
| 188 | return makeConstIterator(TheBucket, |
| 189 | shouldReverseIterate<KeyT>() ? getBuckets() |
| 190 | : getBucketsEnd(), |
| 191 | *this, true); |
| 192 | return end(); |
| 193 | } |
| 194 | |
| 195 | /// lookup - Return the entry for the specified key, or a default |
| 196 | /// constructed value if no such entry exists. |
| 197 | ValueT lookup(const_arg_type_t<KeyT> Val) const { |
| 198 | const BucketT *TheBucket; |
| 199 | if (LookupBucketFor(Val, TheBucket)) |
| 200 | return TheBucket->getSecond(); |
| 201 | return ValueT(); |
| 202 | } |
| 203 | |
| 204 | // Inserts key,value pair into the map if the key isn't already in the map. |
| 205 | // If the key is already in the map, it returns false and doesn't update the |
| 206 | // value. |
| 207 | std::pair<iterator, bool> insert(const std::pair<KeyT, ValueT> &KV) { |
| 208 | return try_emplace(KV.first, KV.second); |
| 209 | } |
| 210 | |
| 211 | // Inserts key,value pair into the map if the key isn't already in the map. |
| 212 | // If the key is already in the map, it returns false and doesn't update the |
| 213 | // value. |
| 214 | std::pair<iterator, bool> insert(std::pair<KeyT, ValueT> &&KV) { |
| 215 | return try_emplace(std::move(KV.first), std::move(KV.second)); |
| 216 | } |
| 217 | |
| 218 | // Inserts key,value pair into the map if the key isn't already in the map. |
| 219 | // The value is constructed in-place if the key is not in the map, otherwise |
| 220 | // it is not moved. |
| 221 | template <typename... Ts> |
| 222 | std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&... Args) { |
| 223 | BucketT *TheBucket; |
| 224 | if (LookupBucketFor(Key, TheBucket)) |
| 225 | return std::make_pair(makeIterator(TheBucket, |
| 226 | shouldReverseIterate<KeyT>() |
| 227 | ? getBuckets() |
| 228 | : getBucketsEnd(), |
| 229 | *this, true), |
| 230 | false); // Already in map. |
| 231 | |
| 232 | // Otherwise, insert the new element. |
| 233 | TheBucket = |
| 234 | InsertIntoBucket(TheBucket, std::move(Key), std::forward<Ts>(Args)...); |
| 235 | return std::make_pair(makeIterator(TheBucket, |
| 236 | shouldReverseIterate<KeyT>() |
| 237 | ? getBuckets() |
| 238 | : getBucketsEnd(), |
| 239 | *this, true), |
| 240 | true); |
| 241 | } |
| 242 | |
| 243 | // Inserts key,value pair into the map if the key isn't already in the map. |
| 244 | // The value is constructed in-place if the key is not in the map, otherwise |
| 245 | // it is not moved. |
| 246 | template <typename... Ts> |
| 247 | std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&... Args) { |
| 248 | BucketT *TheBucket; |
| 249 | if (LookupBucketFor(Key, TheBucket)) |
| 250 | return std::make_pair(makeIterator(TheBucket, |
| 251 | shouldReverseIterate<KeyT>() |
| 252 | ? getBuckets() |
| 253 | : getBucketsEnd(), |
| 254 | *this, true), |
| 255 | false); // Already in map. |
| 256 | |
| 257 | // Otherwise, insert the new element. |
| 258 | TheBucket = InsertIntoBucket(TheBucket, Key, std::forward<Ts>(Args)...); |
| 259 | return std::make_pair(makeIterator(TheBucket, |
| 260 | shouldReverseIterate<KeyT>() |
| 261 | ? getBuckets() |
| 262 | : getBucketsEnd(), |
| 263 | *this, true), |
| 264 | true); |
| 265 | } |
| 266 | |
| 267 | /// Alternate version of insert() which allows a different, and possibly |
| 268 | /// less expensive, key type. |
| 269 | /// The DenseMapInfo is responsible for supplying methods |
| 270 | /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key |
| 271 | /// type used. |
| 272 | template <typename LookupKeyT> |
| 273 | std::pair<iterator, bool> insert_as(std::pair<KeyT, ValueT> &&KV, |
| 274 | const LookupKeyT &Val) { |
| 275 | BucketT *TheBucket; |
| 276 | if (LookupBucketFor(Val, TheBucket)) |
| 277 | return std::make_pair(makeIterator(TheBucket, |
| 278 | shouldReverseIterate<KeyT>() |
| 279 | ? getBuckets() |
| 280 | : getBucketsEnd(), |
| 281 | *this, true), |
| 282 | false); // Already in map. |
| 283 | |
| 284 | // Otherwise, insert the new element. |
| 285 | TheBucket = InsertIntoBucketWithLookup(TheBucket, std::move(KV.first), |
| 286 | std::move(KV.second), Val); |
| 287 | return std::make_pair(makeIterator(TheBucket, |
| 288 | shouldReverseIterate<KeyT>() |
| 289 | ? getBuckets() |
| 290 | : getBucketsEnd(), |
| 291 | *this, true), |
| 292 | true); |
| 293 | } |
| 294 | |
| 295 | /// insert - Range insertion of pairs. |
| 296 | template<typename InputIt> |
| 297 | void insert(InputIt I, InputIt E) { |
| 298 | for (; I != E; ++I) |
| 299 | insert(*I); |
| 300 | } |
| 301 | |
| 302 | bool erase(const KeyT &Val) { |
| 303 | BucketT *TheBucket; |
| 304 | if (!LookupBucketFor(Val, TheBucket)) |
| 305 | return false; // not in map. |
| 306 | |
| 307 | TheBucket->getSecond().~ValueT(); |
| 308 | TheBucket->getFirst() = getTombstoneKey(); |
| 309 | decrementNumEntries(); |
| 310 | incrementNumTombstones(); |
| 311 | return true; |
| 312 | } |
| 313 | void erase(iterator I) { |
| 314 | BucketT *TheBucket = &*I; |
| 315 | TheBucket->getSecond().~ValueT(); |
| 316 | TheBucket->getFirst() = getTombstoneKey(); |
| 317 | decrementNumEntries(); |
| 318 | incrementNumTombstones(); |
| 319 | } |
| 320 | |
| 321 | value_type& FindAndConstruct(const KeyT &Key) { |
| 322 | BucketT *TheBucket; |
| 323 | if (LookupBucketFor(Key, TheBucket)) |
| 324 | return *TheBucket; |
| 325 | |
| 326 | return *InsertIntoBucket(TheBucket, Key); |
| 327 | } |
| 328 | |
| 329 | ValueT &operator[](const KeyT &Key) { |
| 330 | return FindAndConstruct(Key).second; |
| 331 | } |
| 332 | |
| 333 | value_type& FindAndConstruct(KeyT &&Key) { |
| 334 | BucketT *TheBucket; |
| 335 | if (LookupBucketFor(Key, TheBucket)) |
| 336 | return *TheBucket; |
| 337 | |
| 338 | return *InsertIntoBucket(TheBucket, std::move(Key)); |
| 339 | } |
| 340 | |
| 341 | ValueT &operator[](KeyT &&Key) { |
| 342 | return FindAndConstruct(std::move(Key)).second; |
| 343 | } |
| 344 | |
| 345 | /// isPointerIntoBucketsArray - Return true if the specified pointer points |
| 346 | /// somewhere into the DenseMap's array of buckets (i.e. either to a key or |
| 347 | /// value in the DenseMap). |
| 348 | bool isPointerIntoBucketsArray(const void *Ptr) const { |
| 349 | return Ptr >= getBuckets() && Ptr < getBucketsEnd(); |
| 350 | } |
| 351 | |
| 352 | /// getPointerIntoBucketsArray() - Return an opaque pointer into the buckets |
| 353 | /// array. In conjunction with the previous method, this can be used to |
| 354 | /// determine whether an insertion caused the DenseMap to reallocate. |
| 355 | const void *getPointerIntoBucketsArray() const { return getBuckets(); } |
| 356 | |
| 357 | protected: |
| 358 | DenseMapBase() = default; |
| 359 | |
| 360 | void destroyAll() { |
| 361 | if (getNumBuckets() == 0) // Nothing to do. |
| 362 | return; |
| 363 | |
| 364 | const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); |
| 365 | for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { |
| 366 | if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) && |
| 367 | !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) |
| 368 | P->getSecond().~ValueT(); |
| 369 | P->getFirst().~KeyT(); |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | void initEmpty() { |
| 374 | setNumEntries(0); |
| 375 | setNumTombstones(0); |
| 376 | |
| 377 | assert((getNumBuckets() & (getNumBuckets()-1)) == 0 &&(static_cast <bool> ((getNumBuckets() & (getNumBuckets ()-1)) == 0 && "# initial buckets must be a power of two!" ) ? void (0) : __assert_fail ("(getNumBuckets() & (getNumBuckets()-1)) == 0 && \"# initial buckets must be a power of two!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 378, __extension__ __PRETTY_FUNCTION__)) |
| 378 | "# initial buckets must be a power of two!")(static_cast <bool> ((getNumBuckets() & (getNumBuckets ()-1)) == 0 && "# initial buckets must be a power of two!" ) ? void (0) : __assert_fail ("(getNumBuckets() & (getNumBuckets()-1)) == 0 && \"# initial buckets must be a power of two!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 378, __extension__ __PRETTY_FUNCTION__)); |
| 379 | const KeyT EmptyKey = getEmptyKey(); |
| 380 | for (BucketT *B = getBuckets(), *E = getBucketsEnd(); B != E; ++B) |
| 381 | ::new (&B->getFirst()) KeyT(EmptyKey); |
| 382 | } |
| 383 | |
| 384 | /// Returns the number of buckets to allocate to ensure that the DenseMap can |
| 385 | /// accommodate \p NumEntries without need to grow(). |
| 386 | unsigned getMinBucketToReserveForEntries(unsigned NumEntries) { |
| 387 | // Ensure that "NumEntries * 4 < NumBuckets * 3" |
| 388 | if (NumEntries == 0) |
| 389 | return 0; |
| 390 | // +1 is required because of the strict equality. |
| 391 | // For example if NumEntries is 48, we need to return 401. |
| 392 | return NextPowerOf2(NumEntries * 4 / 3 + 1); |
| 393 | } |
| 394 | |
| 395 | void moveFromOldBuckets(BucketT *OldBucketsBegin, BucketT *OldBucketsEnd) { |
| 396 | initEmpty(); |
| 397 | |
| 398 | // Insert all the old elements. |
| 399 | const KeyT EmptyKey = getEmptyKey(); |
| 400 | const KeyT TombstoneKey = getTombstoneKey(); |
| 401 | for (BucketT *B = OldBucketsBegin, *E = OldBucketsEnd; B != E; ++B) { |
| 402 | if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) && |
| 403 | !KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) { |
| 404 | // Insert the key/value into the new table. |
| 405 | BucketT *DestBucket; |
| 406 | bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket); |
| 407 | (void)FoundVal; // silence warning. |
| 408 | assert(!FoundVal && "Key already in new map?")(static_cast <bool> (!FoundVal && "Key already in new map?" ) ? void (0) : __assert_fail ("!FoundVal && \"Key already in new map?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 408, __extension__ __PRETTY_FUNCTION__)); |
| 409 | DestBucket->getFirst() = std::move(B->getFirst()); |
| 410 | ::new (&DestBucket->getSecond()) ValueT(std::move(B->getSecond())); |
| 411 | incrementNumEntries(); |
| 412 | |
| 413 | // Free the value. |
| 414 | B->getSecond().~ValueT(); |
| 415 | } |
| 416 | B->getFirst().~KeyT(); |
| 417 | } |
| 418 | } |
| 419 | |
| 420 | template <typename OtherBaseT> |
| 421 | void copyFrom( |
| 422 | const DenseMapBase<OtherBaseT, KeyT, ValueT, KeyInfoT, BucketT> &other) { |
| 423 | assert(&other != this)(static_cast <bool> (&other != this) ? void (0) : __assert_fail ("&other != this", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 423, __extension__ __PRETTY_FUNCTION__)); |
| 424 | assert(getNumBuckets() == other.getNumBuckets())(static_cast <bool> (getNumBuckets() == other.getNumBuckets ()) ? void (0) : __assert_fail ("getNumBuckets() == other.getNumBuckets()" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 424, __extension__ __PRETTY_FUNCTION__)); |
| 425 | |
| 426 | setNumEntries(other.getNumEntries()); |
| 427 | setNumTombstones(other.getNumTombstones()); |
| 428 | |
| 429 | if (std::is_trivially_copyable<KeyT>::value && |
| 430 | std::is_trivially_copyable<ValueT>::value) |
| 431 | memcpy(reinterpret_cast<void *>(getBuckets()), other.getBuckets(), |
| 432 | getNumBuckets() * sizeof(BucketT)); |
| 433 | else |
| 434 | for (size_t i = 0; i < getNumBuckets(); ++i) { |
| 435 | ::new (&getBuckets()[i].getFirst()) |
| 436 | KeyT(other.getBuckets()[i].getFirst()); |
| 437 | if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) && |
| 438 | !KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey())) |
| 439 | ::new (&getBuckets()[i].getSecond()) |
| 440 | ValueT(other.getBuckets()[i].getSecond()); |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | static unsigned getHashValue(const KeyT &Val) { |
| 445 | return KeyInfoT::getHashValue(Val); |
| 446 | } |
| 447 | |
| 448 | template<typename LookupKeyT> |
| 449 | static unsigned getHashValue(const LookupKeyT &Val) { |
| 450 | return KeyInfoT::getHashValue(Val); |
| 451 | } |
| 452 | |
| 453 | static const KeyT getEmptyKey() { |
| 454 | static_assert(std::is_base_of<DenseMapBase, DerivedT>::value, |
| 455 | "Must pass the derived type to this template!"); |
| 456 | return KeyInfoT::getEmptyKey(); |
| 457 | } |
| 458 | |
| 459 | static const KeyT getTombstoneKey() { |
| 460 | return KeyInfoT::getTombstoneKey(); |
| 461 | } |
| 462 | |
| 463 | private: |
| 464 | iterator makeIterator(BucketT *P, BucketT *E, |
| 465 | DebugEpochBase &Epoch, |
| 466 | bool NoAdvance=false) { |
| 467 | if (shouldReverseIterate<KeyT>()) { |
| 468 | BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; |
| 469 | return iterator(B, E, Epoch, NoAdvance); |
| 470 | } |
| 471 | return iterator(P, E, Epoch, NoAdvance); |
| 472 | } |
| 473 | |
| 474 | const_iterator makeConstIterator(const BucketT *P, const BucketT *E, |
| 475 | const DebugEpochBase &Epoch, |
| 476 | const bool NoAdvance=false) const { |
| 477 | if (shouldReverseIterate<KeyT>()) { |
| 478 | const BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; |
| 479 | return const_iterator(B, E, Epoch, NoAdvance); |
| 480 | } |
| 481 | return const_iterator(P, E, Epoch, NoAdvance); |
| 482 | } |
| 483 | |
| 484 | unsigned getNumEntries() const { |
| 485 | return static_cast<const DerivedT *>(this)->getNumEntries(); |
| 486 | } |
| 487 | |
| 488 | void setNumEntries(unsigned Num) { |
| 489 | static_cast<DerivedT *>(this)->setNumEntries(Num); |
| 490 | } |
| 491 | |
| 492 | void incrementNumEntries() { |
| 493 | setNumEntries(getNumEntries() + 1); |
| 494 | } |
| 495 | |
| 496 | void decrementNumEntries() { |
| 497 | setNumEntries(getNumEntries() - 1); |
| 498 | } |
| 499 | |
| 500 | unsigned getNumTombstones() const { |
| 501 | return static_cast<const DerivedT *>(this)->getNumTombstones(); |
| 502 | } |
| 503 | |
| 504 | void setNumTombstones(unsigned Num) { |
| 505 | static_cast<DerivedT *>(this)->setNumTombstones(Num); |
| 506 | } |
| 507 | |
| 508 | void incrementNumTombstones() { |
| 509 | setNumTombstones(getNumTombstones() + 1); |
| 510 | } |
| 511 | |
| 512 | void decrementNumTombstones() { |
| 513 | setNumTombstones(getNumTombstones() - 1); |
| 514 | } |
| 515 | |
| 516 | const BucketT *getBuckets() const { |
| 517 | return static_cast<const DerivedT *>(this)->getBuckets(); |
| 518 | } |
| 519 | |
| 520 | BucketT *getBuckets() { |
| 521 | return static_cast<DerivedT *>(this)->getBuckets(); |
| 522 | } |
| 523 | |
| 524 | unsigned getNumBuckets() const { |
| 525 | return static_cast<const DerivedT *>(this)->getNumBuckets(); |
| 526 | } |
| 527 | |
| 528 | BucketT *getBucketsEnd() { |
| 529 | return getBuckets() + getNumBuckets(); |
| 530 | } |
| 531 | |
| 532 | const BucketT *getBucketsEnd() const { |
| 533 | return getBuckets() + getNumBuckets(); |
| 534 | } |
| 535 | |
| 536 | void grow(unsigned AtLeast) { |
| 537 | static_cast<DerivedT *>(this)->grow(AtLeast); |
| 538 | } |
| 539 | |
| 540 | void shrink_and_clear() { |
| 541 | static_cast<DerivedT *>(this)->shrink_and_clear(); |
| 542 | } |
| 543 | |
| 544 | template <typename KeyArg, typename... ValueArgs> |
| 545 | BucketT *InsertIntoBucket(BucketT *TheBucket, KeyArg &&Key, |
| 546 | ValueArgs &&... Values) { |
| 547 | TheBucket = InsertIntoBucketImpl(Key, Key, TheBucket); |
| 548 | |
| 549 | TheBucket->getFirst() = std::forward<KeyArg>(Key); |
| 550 | ::new (&TheBucket->getSecond()) ValueT(std::forward<ValueArgs>(Values)...); |
| 551 | return TheBucket; |
| 552 | } |
| 553 | |
| 554 | template <typename LookupKeyT> |
| 555 | BucketT *InsertIntoBucketWithLookup(BucketT *TheBucket, KeyT &&Key, |
| 556 | ValueT &&Value, LookupKeyT &Lookup) { |
| 557 | TheBucket = InsertIntoBucketImpl(Key, Lookup, TheBucket); |
| 558 | |
| 559 | TheBucket->getFirst() = std::move(Key); |
| 560 | ::new (&TheBucket->getSecond()) ValueT(std::move(Value)); |
| 561 | return TheBucket; |
| 562 | } |
| 563 | |
| 564 | template <typename LookupKeyT> |
| 565 | BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup, |
| 566 | BucketT *TheBucket) { |
| 567 | incrementEpoch(); |
| 568 | |
| 569 | // If the load of the hash table is more than 3/4, or if fewer than 1/8 of |
| 570 | // the buckets are empty (meaning that many are filled with tombstones), |
| 571 | // grow the table. |
| 572 | // |
| 573 | // The later case is tricky. For example, if we had one empty bucket with |
| 574 | // tons of tombstones, failing lookups (e.g. for insertion) would have to |
| 575 | // probe almost the entire table until it found the empty bucket. If the |
| 576 | // table completely filled with tombstones, no lookup would ever succeed, |
| 577 | // causing infinite loops in lookup. |
| 578 | unsigned NewNumEntries = getNumEntries() + 1; |
| 579 | unsigned NumBuckets = getNumBuckets(); |
| 580 | if (LLVM_UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)__builtin_expect((bool)(NewNumEntries * 4 >= NumBuckets * 3 ), false)) { |
| 581 | this->grow(NumBuckets * 2); |
| 582 | LookupBucketFor(Lookup, TheBucket); |
| 583 | NumBuckets = getNumBuckets(); |
| 584 | } else if (LLVM_UNLIKELY(NumBuckets-(NewNumEntries+getNumTombstones()) <=__builtin_expect((bool)(NumBuckets-(NewNumEntries+getNumTombstones ()) <= NumBuckets/8), false) |
| 585 | NumBuckets/8)__builtin_expect((bool)(NumBuckets-(NewNumEntries+getNumTombstones ()) <= NumBuckets/8), false)) { |
| 586 | this->grow(NumBuckets); |
| 587 | LookupBucketFor(Lookup, TheBucket); |
| 588 | } |
| 589 | assert(TheBucket)(static_cast <bool> (TheBucket) ? void (0) : __assert_fail ("TheBucket", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 589, __extension__ __PRETTY_FUNCTION__)); |
| 590 | |
| 591 | // Only update the state after we've grown our bucket space appropriately |
| 592 | // so that when growing buckets we have self-consistent entry count. |
| 593 | incrementNumEntries(); |
| 594 | |
| 595 | // If we are writing over a tombstone, remember this. |
| 596 | const KeyT EmptyKey = getEmptyKey(); |
| 597 | if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey)) |
| 598 | decrementNumTombstones(); |
| 599 | |
| 600 | return TheBucket; |
| 601 | } |
| 602 | |
| 603 | /// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in |
| 604 | /// FoundBucket. If the bucket contains the key and a value, this returns |
| 605 | /// true, otherwise it returns a bucket with an empty marker or tombstone and |
| 606 | /// returns false. |
| 607 | template<typename LookupKeyT> |
| 608 | bool LookupBucketFor(const LookupKeyT &Val, |
| 609 | const BucketT *&FoundBucket) const { |
| 610 | const BucketT *BucketsPtr = getBuckets(); |
| 611 | const unsigned NumBuckets = getNumBuckets(); |
| 612 | |
| 613 | if (NumBuckets == 0) { |
| 614 | FoundBucket = nullptr; |
| 615 | return false; |
| 616 | } |
| 617 | |
| 618 | // FoundTombstone - Keep track of whether we find a tombstone while probing. |
| 619 | const BucketT *FoundTombstone = nullptr; |
| 620 | const KeyT EmptyKey = getEmptyKey(); |
| 621 | const KeyT TombstoneKey = getTombstoneKey(); |
| 622 | assert(!KeyInfoT::isEqual(Val, EmptyKey) &&(static_cast <bool> (!KeyInfoT::isEqual(Val, EmptyKey) && !KeyInfoT::isEqual(Val, TombstoneKey) && "Empty/Tombstone value shouldn't be inserted into map!" ) ? void (0) : __assert_fail ("!KeyInfoT::isEqual(Val, EmptyKey) && !KeyInfoT::isEqual(Val, TombstoneKey) && \"Empty/Tombstone value shouldn't be inserted into map!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 624, __extension__ __PRETTY_FUNCTION__)) |
| 623 | !KeyInfoT::isEqual(Val, TombstoneKey) &&(static_cast <bool> (!KeyInfoT::isEqual(Val, EmptyKey) && !KeyInfoT::isEqual(Val, TombstoneKey) && "Empty/Tombstone value shouldn't be inserted into map!" ) ? void (0) : __assert_fail ("!KeyInfoT::isEqual(Val, EmptyKey) && !KeyInfoT::isEqual(Val, TombstoneKey) && \"Empty/Tombstone value shouldn't be inserted into map!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 624, __extension__ __PRETTY_FUNCTION__)) |
| 624 | "Empty/Tombstone value shouldn't be inserted into map!")(static_cast <bool> (!KeyInfoT::isEqual(Val, EmptyKey) && !KeyInfoT::isEqual(Val, TombstoneKey) && "Empty/Tombstone value shouldn't be inserted into map!" ) ? void (0) : __assert_fail ("!KeyInfoT::isEqual(Val, EmptyKey) && !KeyInfoT::isEqual(Val, TombstoneKey) && \"Empty/Tombstone value shouldn't be inserted into map!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 624, __extension__ __PRETTY_FUNCTION__)); |
| 625 | |
| 626 | unsigned BucketNo = getHashValue(Val) & (NumBuckets-1); |
| 627 | unsigned ProbeAmt = 1; |
| 628 | while (true) { |
| 629 | const BucketT *ThisBucket = BucketsPtr + BucketNo; |
| 630 | // Found Val's bucket? If so, return it. |
| 631 | if (LLVM_LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))__builtin_expect((bool)(KeyInfoT::isEqual(Val, ThisBucket-> getFirst())), true)) { |
| 632 | FoundBucket = ThisBucket; |
| 633 | return true; |
| 634 | } |
| 635 | |
| 636 | // If we found an empty bucket, the key doesn't exist in the set. |
| 637 | // Insert it and return the default value. |
| 638 | if (LLVM_LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))__builtin_expect((bool)(KeyInfoT::isEqual(ThisBucket->getFirst (), EmptyKey)), true)) { |
| 639 | // If we've already seen a tombstone while probing, fill it in instead |
| 640 | // of the empty bucket we eventually probed to. |
| 641 | FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket; |
| 642 | return false; |
| 643 | } |
| 644 | |
| 645 | // If this is a tombstone, remember it. If Val ends up not in the map, we |
| 646 | // prefer to return it than something that would require more probing. |
| 647 | if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) && |
| 648 | !FoundTombstone) |
| 649 | FoundTombstone = ThisBucket; // Remember the first tombstone found. |
| 650 | |
| 651 | // Otherwise, it's a hash collision or a tombstone, continue quadratic |
| 652 | // probing. |
| 653 | BucketNo += ProbeAmt++; |
| 654 | BucketNo &= (NumBuckets-1); |
| 655 | } |
| 656 | } |
| 657 | |
| 658 | template <typename LookupKeyT> |
| 659 | bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) { |
| 660 | const BucketT *ConstFoundBucket; |
| 661 | bool Result = const_cast<const DenseMapBase *>(this) |
| 662 | ->LookupBucketFor(Val, ConstFoundBucket); |
| 663 | FoundBucket = const_cast<BucketT *>(ConstFoundBucket); |
| 664 | return Result; |
| 665 | } |
| 666 | |
| 667 | public: |
| 668 | /// Return the approximate size (in bytes) of the actual map. |
| 669 | /// This is just the raw memory used by DenseMap. |
| 670 | /// If entries are pointers to objects, the size of the referenced objects |
| 671 | /// are not included. |
| 672 | size_t getMemorySize() const { |
| 673 | return getNumBuckets() * sizeof(BucketT); |
| 674 | } |
| 675 | }; |
| 676 | |
| 677 | /// Equality comparison for DenseMap. |
| 678 | /// |
| 679 | /// Iterates over elements of LHS confirming that each (key, value) pair in LHS |
| 680 | /// is also in RHS, and that no additional pairs are in RHS. |
| 681 | /// Equivalent to N calls to RHS.find and N value comparisons. Amortized |
| 682 | /// complexity is linear, worst case is O(N^2) (if every hash collides). |
| 683 | template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT, |
| 684 | typename BucketT> |
| 685 | bool operator==( |
| 686 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS, |
| 687 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) { |
| 688 | if (LHS.size() != RHS.size()) |
| 689 | return false; |
| 690 | |
| 691 | for (auto &KV : LHS) { |
| 692 | auto I = RHS.find(KV.first); |
| 693 | if (I == RHS.end() || I->second != KV.second) |
| 694 | return false; |
| 695 | } |
| 696 | |
| 697 | return true; |
| 698 | } |
| 699 | |
| 700 | /// Inequality comparison for DenseMap. |
| 701 | /// |
| 702 | /// Equivalent to !(LHS == RHS). See operator== for performance notes. |
| 703 | template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT, |
| 704 | typename BucketT> |
| 705 | bool operator!=( |
| 706 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS, |
| 707 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) { |
| 708 | return !(LHS == RHS); |
| 709 | } |
| 710 | |
| 711 | template <typename KeyT, typename ValueT, |
| 712 | typename KeyInfoT = DenseMapInfo<KeyT>, |
| 713 | typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>> |
| 714 | class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>, |
| 715 | KeyT, ValueT, KeyInfoT, BucketT> { |
| 716 | friend class DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
| 717 | |
| 718 | // Lift some types from the dependent base class into this class for |
| 719 | // simplicity of referring to them. |
| 720 | using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
| 721 | |
| 722 | BucketT *Buckets; |
| 723 | unsigned NumEntries; |
| 724 | unsigned NumTombstones; |
| 725 | unsigned NumBuckets; |
| 726 | |
| 727 | public: |
| 728 | /// Create a DenseMap with an optional \p InitialReserve that guarantee that |
| 729 | /// this number of elements can be inserted in the map without grow() |
| 730 | explicit DenseMap(unsigned InitialReserve = 0) { init(InitialReserve); } |
| 731 | |
| 732 | DenseMap(const DenseMap &other) : BaseT() { |
| 733 | init(0); |
| 734 | copyFrom(other); |
| 735 | } |
| 736 | |
| 737 | DenseMap(DenseMap &&other) : BaseT() { |
| 738 | init(0); |
| 739 | swap(other); |
| 740 | } |
| 741 | |
| 742 | template<typename InputIt> |
| 743 | DenseMap(const InputIt &I, const InputIt &E) { |
| 744 | init(std::distance(I, E)); |
| 745 | this->insert(I, E); |
| 746 | } |
| 747 | |
| 748 | DenseMap(std::initializer_list<typename BaseT::value_type> Vals) { |
| 749 | init(Vals.size()); |
| 750 | this->insert(Vals.begin(), Vals.end()); |
| 751 | } |
| 752 | |
| 753 | ~DenseMap() { |
| 754 | this->destroyAll(); |
| 755 | deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); |
| 756 | } |
| 757 | |
| 758 | void swap(DenseMap& RHS) { |
| 759 | this->incrementEpoch(); |
| 760 | RHS.incrementEpoch(); |
| 761 | std::swap(Buckets, RHS.Buckets); |
| 762 | std::swap(NumEntries, RHS.NumEntries); |
| 763 | std::swap(NumTombstones, RHS.NumTombstones); |
| 764 | std::swap(NumBuckets, RHS.NumBuckets); |
| 765 | } |
| 766 | |
| 767 | DenseMap& operator=(const DenseMap& other) { |
| 768 | if (&other != this) |
| 769 | copyFrom(other); |
| 770 | return *this; |
| 771 | } |
| 772 | |
| 773 | DenseMap& operator=(DenseMap &&other) { |
| 774 | this->destroyAll(); |
| 775 | deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); |
| 776 | init(0); |
| 777 | swap(other); |
| 778 | return *this; |
| 779 | } |
| 780 | |
| 781 | void copyFrom(const DenseMap& other) { |
| 782 | this->destroyAll(); |
| 783 | deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); |
| 784 | if (allocateBuckets(other.NumBuckets)) { |
| 785 | this->BaseT::copyFrom(other); |
| 786 | } else { |
| 787 | NumEntries = 0; |
| 788 | NumTombstones = 0; |
| 789 | } |
| 790 | } |
| 791 | |
| 792 | void init(unsigned InitNumEntries) { |
| 793 | auto InitBuckets = BaseT::getMinBucketToReserveForEntries(InitNumEntries); |
| 794 | if (allocateBuckets(InitBuckets)) { |
| 795 | this->BaseT::initEmpty(); |
| 796 | } else { |
| 797 | NumEntries = 0; |
| 798 | NumTombstones = 0; |
| 799 | } |
| 800 | } |
| 801 | |
| 802 | void grow(unsigned AtLeast) { |
| 803 | unsigned OldNumBuckets = NumBuckets; |
| 804 | BucketT *OldBuckets = Buckets; |
| 805 | |
| 806 | allocateBuckets(std::max<unsigned>(64, static_cast<unsigned>(NextPowerOf2(AtLeast-1)))); |
| 807 | assert(Buckets)(static_cast <bool> (Buckets) ? void (0) : __assert_fail ("Buckets", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 807, __extension__ __PRETTY_FUNCTION__)); |
| 808 | if (!OldBuckets) { |
| 809 | this->BaseT::initEmpty(); |
| 810 | return; |
| 811 | } |
| 812 | |
| 813 | this->moveFromOldBuckets(OldBuckets, OldBuckets+OldNumBuckets); |
| 814 | |
| 815 | // Free the old table. |
| 816 | deallocate_buffer(OldBuckets, sizeof(BucketT) * OldNumBuckets, |
| 817 | alignof(BucketT)); |
| 818 | } |
| 819 | |
| 820 | void shrink_and_clear() { |
| 821 | unsigned OldNumBuckets = NumBuckets; |
| 822 | unsigned OldNumEntries = NumEntries; |
| 823 | this->destroyAll(); |
| 824 | |
| 825 | // Reduce the number of buckets. |
| 826 | unsigned NewNumBuckets = 0; |
| 827 | if (OldNumEntries) |
| 828 | NewNumBuckets = std::max(64, 1 << (Log2_32_Ceil(OldNumEntries) + 1)); |
| 829 | if (NewNumBuckets == NumBuckets) { |
| 830 | this->BaseT::initEmpty(); |
| 831 | return; |
| 832 | } |
| 833 | |
| 834 | deallocate_buffer(Buckets, sizeof(BucketT) * OldNumBuckets, |
| 835 | alignof(BucketT)); |
| 836 | init(NewNumBuckets); |
| 837 | } |
| 838 | |
| 839 | private: |
| 840 | unsigned getNumEntries() const { |
| 841 | return NumEntries; |
| 842 | } |
| 843 | |
| 844 | void setNumEntries(unsigned Num) { |
| 845 | NumEntries = Num; |
| 846 | } |
| 847 | |
| 848 | unsigned getNumTombstones() const { |
| 849 | return NumTombstones; |
| 850 | } |
| 851 | |
| 852 | void setNumTombstones(unsigned Num) { |
| 853 | NumTombstones = Num; |
| 854 | } |
| 855 | |
| 856 | BucketT *getBuckets() const { |
| 857 | return Buckets; |
| 858 | } |
| 859 | |
| 860 | unsigned getNumBuckets() const { |
| 861 | return NumBuckets; |
| 862 | } |
| 863 | |
| 864 | bool allocateBuckets(unsigned Num) { |
| 865 | NumBuckets = Num; |
| 866 | if (NumBuckets == 0) { |
| 867 | Buckets = nullptr; |
| 868 | return false; |
| 869 | } |
| 870 | |
| 871 | Buckets = static_cast<BucketT *>( |
| 872 | allocate_buffer(sizeof(BucketT) * NumBuckets, alignof(BucketT))); |
| 873 | return true; |
| 874 | } |
| 875 | }; |
| 876 | |
| 877 | template <typename KeyT, typename ValueT, unsigned InlineBuckets = 4, |
| 878 | typename KeyInfoT = DenseMapInfo<KeyT>, |
| 879 | typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>> |
| 880 | class SmallDenseMap |
| 881 | : public DenseMapBase< |
| 882 | SmallDenseMap<KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT>, KeyT, |
| 883 | ValueT, KeyInfoT, BucketT> { |
| 884 | friend class DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
| 885 | |
| 886 | // Lift some types from the dependent base class into this class for |
| 887 | // simplicity of referring to them. |
| 888 | using BaseT = DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
| 889 | |
| 890 | static_assert(isPowerOf2_64(InlineBuckets), |
| 891 | "InlineBuckets must be a power of 2."); |
| 892 | |
| 893 | unsigned Small : 1; |
| 894 | unsigned NumEntries : 31; |
| 895 | unsigned NumTombstones; |
| 896 | |
| 897 | struct LargeRep { |
| 898 | BucketT *Buckets; |
| 899 | unsigned NumBuckets; |
| 900 | }; |
| 901 | |
| 902 | /// A "union" of an inline bucket array and the struct representing |
| 903 | /// a large bucket. This union will be discriminated by the 'Small' bit. |
| 904 | AlignedCharArrayUnion<BucketT[InlineBuckets], LargeRep> storage; |
| 905 | |
| 906 | public: |
| 907 | explicit SmallDenseMap(unsigned NumInitBuckets = 0) { |
| 908 | init(NumInitBuckets); |
| 909 | } |
| 910 | |
| 911 | SmallDenseMap(const SmallDenseMap &other) : BaseT() { |
| 912 | init(0); |
| 913 | copyFrom(other); |
| 914 | } |
| 915 | |
| 916 | SmallDenseMap(SmallDenseMap &&other) : BaseT() { |
| 917 | init(0); |
| 918 | swap(other); |
| 919 | } |
| 920 | |
| 921 | template<typename InputIt> |
| 922 | SmallDenseMap(const InputIt &I, const InputIt &E) { |
| 923 | init(NextPowerOf2(std::distance(I, E))); |
| 924 | this->insert(I, E); |
| 925 | } |
| 926 | |
| 927 | SmallDenseMap(std::initializer_list<typename BaseT::value_type> Vals) |
| 928 | : SmallDenseMap(Vals.begin(), Vals.end()) {} |
| 929 | |
| 930 | ~SmallDenseMap() { |
| 931 | this->destroyAll(); |
| 932 | deallocateBuckets(); |
| 933 | } |
| 934 | |
| 935 | void swap(SmallDenseMap& RHS) { |
| 936 | unsigned TmpNumEntries = RHS.NumEntries; |
| 937 | RHS.NumEntries = NumEntries; |
| 938 | NumEntries = TmpNumEntries; |
| 939 | std::swap(NumTombstones, RHS.NumTombstones); |
| 940 | |
| 941 | const KeyT EmptyKey = this->getEmptyKey(); |
| 942 | const KeyT TombstoneKey = this->getTombstoneKey(); |
| 943 | if (Small && RHS.Small) { |
| 944 | // If we're swapping inline bucket arrays, we have to cope with some of |
| 945 | // the tricky bits of DenseMap's storage system: the buckets are not |
| 946 | // fully initialized. Thus we swap every key, but we may have |
| 947 | // a one-directional move of the value. |
| 948 | for (unsigned i = 0, e = InlineBuckets; i != e; ++i) { |
| 949 | BucketT *LHSB = &getInlineBuckets()[i], |
| 950 | *RHSB = &RHS.getInlineBuckets()[i]; |
| 951 | bool hasLHSValue = (!KeyInfoT::isEqual(LHSB->getFirst(), EmptyKey) && |
| 952 | !KeyInfoT::isEqual(LHSB->getFirst(), TombstoneKey)); |
| 953 | bool hasRHSValue = (!KeyInfoT::isEqual(RHSB->getFirst(), EmptyKey) && |
| 954 | !KeyInfoT::isEqual(RHSB->getFirst(), TombstoneKey)); |
| 955 | if (hasLHSValue && hasRHSValue) { |
| 956 | // Swap together if we can... |
| 957 | std::swap(*LHSB, *RHSB); |
| 958 | continue; |
| 959 | } |
| 960 | // Swap separately and handle any asymmetry. |
| 961 | std::swap(LHSB->getFirst(), RHSB->getFirst()); |
| 962 | if (hasLHSValue) { |
| 963 | ::new (&RHSB->getSecond()) ValueT(std::move(LHSB->getSecond())); |
| 964 | LHSB->getSecond().~ValueT(); |
| 965 | } else if (hasRHSValue) { |
| 966 | ::new (&LHSB->getSecond()) ValueT(std::move(RHSB->getSecond())); |
| 967 | RHSB->getSecond().~ValueT(); |
| 968 | } |
| 969 | } |
| 970 | return; |
| 971 | } |
| 972 | if (!Small && !RHS.Small) { |
| 973 | std::swap(getLargeRep()->Buckets, RHS.getLargeRep()->Buckets); |
| 974 | std::swap(getLargeRep()->NumBuckets, RHS.getLargeRep()->NumBuckets); |
| 975 | return; |
| 976 | } |
| 977 | |
| 978 | SmallDenseMap &SmallSide = Small ? *this : RHS; |
| 979 | SmallDenseMap &LargeSide = Small ? RHS : *this; |
| 980 | |
| 981 | // First stash the large side's rep and move the small side across. |
| 982 | LargeRep TmpRep = std::move(*LargeSide.getLargeRep()); |
| 983 | LargeSide.getLargeRep()->~LargeRep(); |
| 984 | LargeSide.Small = true; |
| 985 | // This is similar to the standard move-from-old-buckets, but the bucket |
| 986 | // count hasn't actually rotated in this case. So we have to carefully |
| 987 | // move construct the keys and values into their new locations, but there |
| 988 | // is no need to re-hash things. |
| 989 | for (unsigned i = 0, e = InlineBuckets; i != e; ++i) { |
| 990 | BucketT *NewB = &LargeSide.getInlineBuckets()[i], |
| 991 | *OldB = &SmallSide.getInlineBuckets()[i]; |
| 992 | ::new (&NewB->getFirst()) KeyT(std::move(OldB->getFirst())); |
| 993 | OldB->getFirst().~KeyT(); |
| 994 | if (!KeyInfoT::isEqual(NewB->getFirst(), EmptyKey) && |
| 995 | !KeyInfoT::isEqual(NewB->getFirst(), TombstoneKey)) { |
| 996 | ::new (&NewB->getSecond()) ValueT(std::move(OldB->getSecond())); |
| 997 | OldB->getSecond().~ValueT(); |
| 998 | } |
| 999 | } |
| 1000 | |
| 1001 | // The hard part of moving the small buckets across is done, just move |
| 1002 | // the TmpRep into its new home. |
| 1003 | SmallSide.Small = false; |
| 1004 | new (SmallSide.getLargeRep()) LargeRep(std::move(TmpRep)); |
| 1005 | } |
| 1006 | |
| 1007 | SmallDenseMap& operator=(const SmallDenseMap& other) { |
| 1008 | if (&other != this) |
| 1009 | copyFrom(other); |
| 1010 | return *this; |
| 1011 | } |
| 1012 | |
| 1013 | SmallDenseMap& operator=(SmallDenseMap &&other) { |
| 1014 | this->destroyAll(); |
| 1015 | deallocateBuckets(); |
| 1016 | init(0); |
| 1017 | swap(other); |
| 1018 | return *this; |
| 1019 | } |
| 1020 | |
| 1021 | void copyFrom(const SmallDenseMap& other) { |
| 1022 | this->destroyAll(); |
| 1023 | deallocateBuckets(); |
| 1024 | Small = true; |
| 1025 | if (other.getNumBuckets() > InlineBuckets) { |
| 1026 | Small = false; |
| 1027 | new (getLargeRep()) LargeRep(allocateBuckets(other.getNumBuckets())); |
| 1028 | } |
| 1029 | this->BaseT::copyFrom(other); |
| 1030 | } |
| 1031 | |
| 1032 | void init(unsigned InitBuckets) { |
| 1033 | Small = true; |
| 1034 | if (InitBuckets > InlineBuckets) { |
| 1035 | Small = false; |
| 1036 | new (getLargeRep()) LargeRep(allocateBuckets(InitBuckets)); |
| 1037 | } |
| 1038 | this->BaseT::initEmpty(); |
| 1039 | } |
| 1040 | |
| 1041 | void grow(unsigned AtLeast) { |
| 1042 | if (AtLeast > InlineBuckets) |
| 1043 | AtLeast = std::max<unsigned>(64, NextPowerOf2(AtLeast-1)); |
| 1044 | |
| 1045 | if (Small) { |
| 1046 | // First move the inline buckets into a temporary storage. |
| 1047 | AlignedCharArrayUnion<BucketT[InlineBuckets]> TmpStorage; |
| 1048 | BucketT *TmpBegin = reinterpret_cast<BucketT *>(&TmpStorage); |
| 1049 | BucketT *TmpEnd = TmpBegin; |
| 1050 | |
| 1051 | // Loop over the buckets, moving non-empty, non-tombstones into the |
| 1052 | // temporary storage. Have the loop move the TmpEnd forward as it goes. |
| 1053 | const KeyT EmptyKey = this->getEmptyKey(); |
| 1054 | const KeyT TombstoneKey = this->getTombstoneKey(); |
| 1055 | for (BucketT *P = getBuckets(), *E = P + InlineBuckets; P != E; ++P) { |
| 1056 | if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) && |
| 1057 | !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { |
| 1058 | assert(size_t(TmpEnd - TmpBegin) < InlineBuckets &&(static_cast <bool> (size_t(TmpEnd - TmpBegin) < InlineBuckets && "Too many inline buckets!") ? void (0) : __assert_fail ("size_t(TmpEnd - TmpBegin) < InlineBuckets && \"Too many inline buckets!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1059, __extension__ __PRETTY_FUNCTION__)) |
| 1059 | "Too many inline buckets!")(static_cast <bool> (size_t(TmpEnd - TmpBegin) < InlineBuckets && "Too many inline buckets!") ? void (0) : __assert_fail ("size_t(TmpEnd - TmpBegin) < InlineBuckets && \"Too many inline buckets!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1059, __extension__ __PRETTY_FUNCTION__)); |
| 1060 | ::new (&TmpEnd->getFirst()) KeyT(std::move(P->getFirst())); |
| 1061 | ::new (&TmpEnd->getSecond()) ValueT(std::move(P->getSecond())); |
| 1062 | ++TmpEnd; |
| 1063 | P->getSecond().~ValueT(); |
| 1064 | } |
| 1065 | P->getFirst().~KeyT(); |
| 1066 | } |
| 1067 | |
| 1068 | // AtLeast == InlineBuckets can happen if there are many tombstones, |
| 1069 | // and grow() is used to remove them. Usually we always switch to the |
| 1070 | // large rep here. |
| 1071 | if (AtLeast > InlineBuckets) { |
| 1072 | Small = false; |
| 1073 | new (getLargeRep()) LargeRep(allocateBuckets(AtLeast)); |
| 1074 | } |
| 1075 | this->moveFromOldBuckets(TmpBegin, TmpEnd); |
| 1076 | return; |
| 1077 | } |
| 1078 | |
| 1079 | LargeRep OldRep = std::move(*getLargeRep()); |
| 1080 | getLargeRep()->~LargeRep(); |
| 1081 | if (AtLeast <= InlineBuckets) { |
| 1082 | Small = true; |
| 1083 | } else { |
| 1084 | new (getLargeRep()) LargeRep(allocateBuckets(AtLeast)); |
| 1085 | } |
| 1086 | |
| 1087 | this->moveFromOldBuckets(OldRep.Buckets, OldRep.Buckets+OldRep.NumBuckets); |
| 1088 | |
| 1089 | // Free the old table. |
| 1090 | deallocate_buffer(OldRep.Buckets, sizeof(BucketT) * OldRep.NumBuckets, |
| 1091 | alignof(BucketT)); |
| 1092 | } |
| 1093 | |
| 1094 | void shrink_and_clear() { |
| 1095 | unsigned OldSize = this->size(); |
| 1096 | this->destroyAll(); |
| 1097 | |
| 1098 | // Reduce the number of buckets. |
| 1099 | unsigned NewNumBuckets = 0; |
| 1100 | if (OldSize) { |
| 1101 | NewNumBuckets = 1 << (Log2_32_Ceil(OldSize) + 1); |
| 1102 | if (NewNumBuckets > InlineBuckets && NewNumBuckets < 64u) |
| 1103 | NewNumBuckets = 64; |
| 1104 | } |
| 1105 | if ((Small && NewNumBuckets <= InlineBuckets) || |
| 1106 | (!Small && NewNumBuckets == getLargeRep()->NumBuckets)) { |
| 1107 | this->BaseT::initEmpty(); |
| 1108 | return; |
| 1109 | } |
| 1110 | |
| 1111 | deallocateBuckets(); |
| 1112 | init(NewNumBuckets); |
| 1113 | } |
| 1114 | |
| 1115 | private: |
| 1116 | unsigned getNumEntries() const { |
| 1117 | return NumEntries; |
| 1118 | } |
| 1119 | |
| 1120 | void setNumEntries(unsigned Num) { |
| 1121 | // NumEntries is hardcoded to be 31 bits wide. |
| 1122 | assert(Num < (1U << 31) && "Cannot support more than 1<<31 entries")(static_cast <bool> (Num < (1U << 31) && "Cannot support more than 1<<31 entries") ? void (0) : __assert_fail ("Num < (1U << 31) && \"Cannot support more than 1<<31 entries\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1122, __extension__ __PRETTY_FUNCTION__)); |
| 1123 | NumEntries = Num; |
| 1124 | } |
| 1125 | |
| 1126 | unsigned getNumTombstones() const { |
| 1127 | return NumTombstones; |
| 1128 | } |
| 1129 | |
| 1130 | void setNumTombstones(unsigned Num) { |
| 1131 | NumTombstones = Num; |
| 1132 | } |
| 1133 | |
| 1134 | const BucketT *getInlineBuckets() const { |
| 1135 | assert(Small)(static_cast <bool> (Small) ? void (0) : __assert_fail ( "Small", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1135, __extension__ __PRETTY_FUNCTION__)); |
| 1136 | // Note that this cast does not violate aliasing rules as we assert that |
| 1137 | // the memory's dynamic type is the small, inline bucket buffer, and the |
| 1138 | // 'storage' is a POD containing a char buffer. |
| 1139 | return reinterpret_cast<const BucketT *>(&storage); |
| 1140 | } |
| 1141 | |
| 1142 | BucketT *getInlineBuckets() { |
| 1143 | return const_cast<BucketT *>( |
| 1144 | const_cast<const SmallDenseMap *>(this)->getInlineBuckets()); |
| 1145 | } |
| 1146 | |
| 1147 | const LargeRep *getLargeRep() const { |
| 1148 | assert(!Small)(static_cast <bool> (!Small) ? void (0) : __assert_fail ("!Small", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1148, __extension__ __PRETTY_FUNCTION__)); |
| 1149 | // Note, same rule about aliasing as with getInlineBuckets. |
| 1150 | return reinterpret_cast<const LargeRep *>(&storage); |
| 1151 | } |
| 1152 | |
| 1153 | LargeRep *getLargeRep() { |
| 1154 | return const_cast<LargeRep *>( |
| 1155 | const_cast<const SmallDenseMap *>(this)->getLargeRep()); |
| 1156 | } |
| 1157 | |
| 1158 | const BucketT *getBuckets() const { |
| 1159 | return Small ? getInlineBuckets() : getLargeRep()->Buckets; |
| 1160 | } |
| 1161 | |
| 1162 | BucketT *getBuckets() { |
| 1163 | return const_cast<BucketT *>( |
| 1164 | const_cast<const SmallDenseMap *>(this)->getBuckets()); |
| 1165 | } |
| 1166 | |
| 1167 | unsigned getNumBuckets() const { |
| 1168 | return Small ? InlineBuckets : getLargeRep()->NumBuckets; |
| 1169 | } |
| 1170 | |
| 1171 | void deallocateBuckets() { |
| 1172 | if (Small) |
| 1173 | return; |
| 1174 | |
| 1175 | deallocate_buffer(getLargeRep()->Buckets, |
| 1176 | sizeof(BucketT) * getLargeRep()->NumBuckets, |
| 1177 | alignof(BucketT)); |
| 1178 | getLargeRep()->~LargeRep(); |
| 1179 | } |
| 1180 | |
| 1181 | LargeRep allocateBuckets(unsigned Num) { |
| 1182 | assert(Num > InlineBuckets && "Must allocate more buckets than are inline")(static_cast <bool> (Num > InlineBuckets && "Must allocate more buckets than are inline" ) ? void (0) : __assert_fail ("Num > InlineBuckets && \"Must allocate more buckets than are inline\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1182, __extension__ __PRETTY_FUNCTION__)); |
| 1183 | LargeRep Rep = {static_cast<BucketT *>(allocate_buffer( |
| 1184 | sizeof(BucketT) * Num, alignof(BucketT))), |
| 1185 | Num}; |
| 1186 | return Rep; |
| 1187 | } |
| 1188 | }; |
| 1189 | |
| 1190 | template <typename KeyT, typename ValueT, typename KeyInfoT, typename Bucket, |
| 1191 | bool IsConst> |
| 1192 | class DenseMapIterator : DebugEpochBase::HandleBase { |
| 1193 | friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>; |
| 1194 | friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, false>; |
| 1195 | |
| 1196 | public: |
| 1197 | using difference_type = ptrdiff_t; |
| 1198 | using value_type = |
| 1199 | typename std::conditional<IsConst, const Bucket, Bucket>::type; |
| 1200 | using pointer = value_type *; |
| 1201 | using reference = value_type &; |
| 1202 | using iterator_category = std::forward_iterator_tag; |
| 1203 | |
| 1204 | private: |
| 1205 | pointer Ptr = nullptr; |
| 1206 | pointer End = nullptr; |
| 1207 | |
| 1208 | public: |
| 1209 | DenseMapIterator() = default; |
| 1210 | |
| 1211 | DenseMapIterator(pointer Pos, pointer E, const DebugEpochBase &Epoch, |
| 1212 | bool NoAdvance = false) |
| 1213 | : DebugEpochBase::HandleBase(&Epoch), Ptr(Pos), End(E) { |
| 1214 | assert(isHandleInSync() && "invalid construction!")(static_cast <bool> (isHandleInSync() && "invalid construction!" ) ? void (0) : __assert_fail ("isHandleInSync() && \"invalid construction!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1214, __extension__ __PRETTY_FUNCTION__)); |
| 1215 | |
| 1216 | if (NoAdvance) return; |
| 1217 | if (shouldReverseIterate<KeyT>()) { |
| 1218 | RetreatPastEmptyBuckets(); |
| 1219 | return; |
| 1220 | } |
| 1221 | AdvancePastEmptyBuckets(); |
| 1222 | } |
| 1223 | |
| 1224 | // Converting ctor from non-const iterators to const iterators. SFINAE'd out |
| 1225 | // for const iterator destinations so it doesn't end up as a user defined copy |
| 1226 | // constructor. |
| 1227 | template <bool IsConstSrc, |
| 1228 | typename = std::enable_if_t<!IsConstSrc && IsConst>> |
| 1229 | DenseMapIterator( |
| 1230 | const DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, IsConstSrc> &I) |
| 1231 | : DebugEpochBase::HandleBase(I), Ptr(I.Ptr), End(I.End) {} |
| 1232 | |
| 1233 | reference operator*() const { |
| 1234 | assert(isHandleInSync() && "invalid iterator access!")(static_cast <bool> (isHandleInSync() && "invalid iterator access!" ) ? void (0) : __assert_fail ("isHandleInSync() && \"invalid iterator access!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1234, __extension__ __PRETTY_FUNCTION__)); |
| 1235 | assert(Ptr != End && "dereferencing end() iterator")(static_cast <bool> (Ptr != End && "dereferencing end() iterator" ) ? void (0) : __assert_fail ("Ptr != End && \"dereferencing end() iterator\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1235, __extension__ __PRETTY_FUNCTION__)); |
| 1236 | if (shouldReverseIterate<KeyT>()) |
| 1237 | return Ptr[-1]; |
| 1238 | return *Ptr; |
| 1239 | } |
| 1240 | pointer operator->() const { |
| 1241 | assert(isHandleInSync() && "invalid iterator access!")(static_cast <bool> (isHandleInSync() && "invalid iterator access!" ) ? void (0) : __assert_fail ("isHandleInSync() && \"invalid iterator access!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1241, __extension__ __PRETTY_FUNCTION__)); |
| 1242 | assert(Ptr != End && "dereferencing end() iterator")(static_cast <bool> (Ptr != End && "dereferencing end() iterator" ) ? void (0) : __assert_fail ("Ptr != End && \"dereferencing end() iterator\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1242, __extension__ __PRETTY_FUNCTION__)); |
| 1243 | if (shouldReverseIterate<KeyT>()) |
| 1244 | return &(Ptr[-1]); |
| 1245 | return Ptr; |
| 1246 | } |
| 1247 | |
| 1248 | friend bool operator==(const DenseMapIterator &LHS, |
| 1249 | const DenseMapIterator &RHS) { |
| 1250 | assert((!LHS.Ptr || LHS.isHandleInSync()) && "handle not in sync!")(static_cast <bool> ((!LHS.Ptr || LHS.isHandleInSync()) && "handle not in sync!") ? void (0) : __assert_fail ("(!LHS.Ptr || LHS.isHandleInSync()) && \"handle not in sync!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1250, __extension__ __PRETTY_FUNCTION__)); |
| 1251 | assert((!RHS.Ptr || RHS.isHandleInSync()) && "handle not in sync!")(static_cast <bool> ((!RHS.Ptr || RHS.isHandleInSync()) && "handle not in sync!") ? void (0) : __assert_fail ("(!RHS.Ptr || RHS.isHandleInSync()) && \"handle not in sync!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1251, __extension__ __PRETTY_FUNCTION__)); |
| 1252 | assert(LHS.getEpochAddress() == RHS.getEpochAddress() &&(static_cast <bool> (LHS.getEpochAddress() == RHS.getEpochAddress () && "comparing incomparable iterators!") ? void (0) : __assert_fail ("LHS.getEpochAddress() == RHS.getEpochAddress() && \"comparing incomparable iterators!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1253, __extension__ __PRETTY_FUNCTION__)) |
| 1253 | "comparing incomparable iterators!")(static_cast <bool> (LHS.getEpochAddress() == RHS.getEpochAddress () && "comparing incomparable iterators!") ? void (0) : __assert_fail ("LHS.getEpochAddress() == RHS.getEpochAddress() && \"comparing incomparable iterators!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1253, __extension__ __PRETTY_FUNCTION__)); |
| 1254 | return LHS.Ptr == RHS.Ptr; |
| 1255 | } |
| 1256 | |
| 1257 | friend bool operator!=(const DenseMapIterator &LHS, |
| 1258 | const DenseMapIterator &RHS) { |
| 1259 | return !(LHS == RHS); |
| 1260 | } |
| 1261 | |
| 1262 | inline DenseMapIterator& operator++() { // Preincrement |
| 1263 | assert(isHandleInSync() && "invalid iterator access!")(static_cast <bool> (isHandleInSync() && "invalid iterator access!" ) ? void (0) : __assert_fail ("isHandleInSync() && \"invalid iterator access!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1263, __extension__ __PRETTY_FUNCTION__)); |
| 1264 | assert(Ptr != End && "incrementing end() iterator")(static_cast <bool> (Ptr != End && "incrementing end() iterator" ) ? void (0) : __assert_fail ("Ptr != End && \"incrementing end() iterator\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1264, __extension__ __PRETTY_FUNCTION__)); |
| 1265 | if (shouldReverseIterate<KeyT>()) { |
| 1266 | --Ptr; |
| 1267 | RetreatPastEmptyBuckets(); |
| 1268 | return *this; |
| 1269 | } |
| 1270 | ++Ptr; |
| 1271 | AdvancePastEmptyBuckets(); |
| 1272 | return *this; |
| 1273 | } |
| 1274 | DenseMapIterator operator++(int) { // Postincrement |
| 1275 | assert(isHandleInSync() && "invalid iterator access!")(static_cast <bool> (isHandleInSync() && "invalid iterator access!" ) ? void (0) : __assert_fail ("isHandleInSync() && \"invalid iterator access!\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1275, __extension__ __PRETTY_FUNCTION__)); |
| 1276 | DenseMapIterator tmp = *this; ++*this; return tmp; |
| 1277 | } |
| 1278 | |
| 1279 | private: |
| 1280 | void AdvancePastEmptyBuckets() { |
| 1281 | assert(Ptr <= End)(static_cast <bool> (Ptr <= End) ? void (0) : __assert_fail ("Ptr <= End", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1281, __extension__ __PRETTY_FUNCTION__)); |
| 1282 | const KeyT Empty = KeyInfoT::getEmptyKey(); |
| 1283 | const KeyT Tombstone = KeyInfoT::getTombstoneKey(); |
| 1284 | |
| 1285 | while (Ptr != End && (KeyInfoT::isEqual(Ptr->getFirst(), Empty) || |
| 1286 | KeyInfoT::isEqual(Ptr->getFirst(), Tombstone))) |
| 1287 | ++Ptr; |
| 1288 | } |
| 1289 | |
| 1290 | void RetreatPastEmptyBuckets() { |
| 1291 | assert(Ptr >= End)(static_cast <bool> (Ptr >= End) ? void (0) : __assert_fail ("Ptr >= End", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/ADT/DenseMap.h" , 1291, __extension__ __PRETTY_FUNCTION__)); |
| 1292 | const KeyT Empty = KeyInfoT::getEmptyKey(); |
| 1293 | const KeyT Tombstone = KeyInfoT::getTombstoneKey(); |
| 1294 | |
| 1295 | while (Ptr != End && (KeyInfoT::isEqual(Ptr[-1].getFirst(), Empty) || |
| 1296 | KeyInfoT::isEqual(Ptr[-1].getFirst(), Tombstone))) |
| 1297 | --Ptr; |
| 1298 | } |
| 1299 | }; |
| 1300 | |
| 1301 | template <typename KeyT, typename ValueT, typename KeyInfoT> |
| 1302 | inline size_t capacity_in_bytes(const DenseMap<KeyT, ValueT, KeyInfoT> &X) { |
| 1303 | return X.getMemorySize(); |
| 1304 | } |
| 1305 | |
| 1306 | } // end namespace llvm |
| 1307 | |
| 1308 | #endif // LLVM_ADT_DENSEMAP_H |